diff --git a/accession-commons-core/src/main/java/uk/ac/ebi/ampt2d/commons/accession/core/AccessionSaveMode.java b/accession-commons-core/src/main/java/uk/ac/ebi/ampt2d/commons/accession/core/AccessionSaveMode.java new file mode 100644 index 00000000..4e8e3536 --- /dev/null +++ b/accession-commons-core/src/main/java/uk/ac/ebi/ampt2d/commons/accession/core/AccessionSaveMode.java @@ -0,0 +1,6 @@ +package uk.ac.ebi.ampt2d.commons.accession.core; + +public enum AccessionSaveMode { + SAVE_ALL_THEN_RESOLVE, + PREFILTER_EXISTING +} diff --git a/accession-commons-core/src/main/java/uk/ac/ebi/ampt2d/commons/accession/core/BasicAccessioningService.java b/accession-commons-core/src/main/java/uk/ac/ebi/ampt2d/commons/accession/core/BasicAccessioningService.java index f09bba06..a46aa8b4 100644 --- a/accession-commons-core/src/main/java/uk/ac/ebi/ampt2d/commons/accession/core/BasicAccessioningService.java +++ b/accession-commons-core/src/main/java/uk/ac/ebi/ampt2d/commons/accession/core/BasicAccessioningService.java @@ -59,13 +59,17 @@ public class BasicAccessioningService hashingFunction; + private final AccessionSaveMode accessionSaveMode; + public BasicAccessioningService(AccessionGenerator accessionGenerator, DatabaseService dbService, Function summaryFunction, - Function hashingFunction) { + Function hashingFunction, + AccessionSaveMode accessionSaveMode) { this.accessionGenerator = accessionGenerator; this.dbService = dbService; this.hashingFunction = summaryFunction.andThen(hashingFunction); + this.accessionSaveMode = accessionSaveMode != null ? accessionSaveMode : AccessionSaveMode.SAVE_ALL_THEN_RESOLVE; } @Override @@ -89,7 +93,17 @@ private Map mapHashOfMessages(List messages) { * application instance / thread with a different id. * See {@link #getPreexistingAccessions(List)} } for more details. */ - private List> saveAccessions( + private List> saveAccessions(List> accessions) { + switch (this.accessionSaveMode) { + case PREFILTER_EXISTING: + return saveAccessionsPrefilteringExisting(accessions); + case SAVE_ALL_THEN_RESOLVE: + default: + return saveAllAccessionsThenResolve(accessions); + } + } + + private List> saveAllAccessionsThenResolve( List> accessions) { SaveResponse response = dbService.save(accessions); accessionGenerator.postSave(response); @@ -110,6 +124,28 @@ private List> saveAccessions return savedAccessions; } + private List> saveAccessionsPrefilteringExisting( + List> accessions) { + Set allHashes = accessions.stream().map(AccessionWrapper::getHash).collect(Collectors.toSet()); + List> preexistingAccessions = dbService.findAllByHash(allHashes); + Set preexistingHashes = preexistingAccessions.stream().map(AccessionWrapper::getHash).collect(Collectors.toSet()); + + List> accessionsToSave = accessions.stream() + .filter(accession -> !preexistingHashes.contains(accession.getHash())) + .collect(Collectors.toList()); + + final List> result = new ArrayList<>(); + + if (!accessionsToSave.isEmpty()) { + result.addAll(saveAllAccessionsThenResolve(accessionsToSave)); + } + + // add pre-existing back to result + preexistingAccessions.stream().map(GetOrCreateAccessionWrapper::oldAccession).forEach(result::add); + + return result; + } + /** * We try to recover all elements that could not be saved to return their accession to the user. This is only * expected when another application instance or thread has saved that element already with a different id. If diff --git a/accession-commons-jpa/src/test/java/uk/ac/ebi/ampt2d/test/configuration/CucumberTestConfiguration.java b/accession-commons-jpa/src/test/java/uk/ac/ebi/ampt2d/test/configuration/CucumberTestConfiguration.java index 022ed530..e798b9a5 100644 --- a/accession-commons-jpa/src/test/java/uk/ac/ebi/ampt2d/test/configuration/CucumberTestConfiguration.java +++ b/accession-commons-jpa/src/test/java/uk/ac/ebi/ampt2d/test/configuration/CucumberTestConfiguration.java @@ -28,6 +28,7 @@ import org.springframework.context.annotation.Scope; import org.springframework.data.jpa.repository.config.EnableJpaAuditing; import org.springframework.data.jpa.repository.config.EnableJpaRepositories; +import uk.ac.ebi.ampt2d.commons.accession.core.AccessionSaveMode; import uk.ac.ebi.ampt2d.commons.accession.core.AccessioningService; import uk.ac.ebi.ampt2d.commons.accession.core.BasicAccessioningService; import uk.ac.ebi.ampt2d.commons.accession.core.DatabaseService; @@ -97,7 +98,8 @@ public AccessioningService accessioningService() { ), databaseService(), TestModel::getValue, - new SHA1HashingFunction() + new SHA1HashingFunction(), + AccessionSaveMode.SAVE_ALL_THEN_RESOLVE ); } diff --git a/accession-commons-jpa/src/test/java/uk/ac/ebi/ampt2d/test/configuration/TestJpaDatabaseServiceTestConfiguration.java b/accession-commons-jpa/src/test/java/uk/ac/ebi/ampt2d/test/configuration/TestJpaDatabaseServiceTestConfiguration.java index c803f703..0ef0ce33 100644 --- a/accession-commons-jpa/src/test/java/uk/ac/ebi/ampt2d/test/configuration/TestJpaDatabaseServiceTestConfiguration.java +++ b/accession-commons-jpa/src/test/java/uk/ac/ebi/ampt2d/test/configuration/TestJpaDatabaseServiceTestConfiguration.java @@ -27,6 +27,7 @@ import org.springframework.context.annotation.Configuration; import org.springframework.data.jpa.repository.config.EnableJpaAuditing; import org.springframework.data.jpa.repository.config.EnableJpaRepositories; +import uk.ac.ebi.ampt2d.commons.accession.core.AccessionSaveMode; import uk.ac.ebi.ampt2d.commons.accession.core.AccessioningService; import uk.ac.ebi.ampt2d.commons.accession.core.BasicAccessioningService; import uk.ac.ebi.ampt2d.commons.accession.generators.SingleAccessionGenerator; @@ -95,7 +96,8 @@ public AccessioningService accessioningService() { ), databaseService(), TestModel::getValue, - new SHA1HashingFunction() + new SHA1HashingFunction(), + AccessionSaveMode.SAVE_ALL_THEN_RESOLVE ); } diff --git a/accession-commons-mongodb/src/test/java/uk/ac/ebi/ampt2d/test/configuration/MongoDbCucumberTestConfiguration.java b/accession-commons-mongodb/src/test/java/uk/ac/ebi/ampt2d/test/configuration/MongoDbCucumberTestConfiguration.java index 11aebc92..50531c6c 100644 --- a/accession-commons-mongodb/src/test/java/uk/ac/ebi/ampt2d/test/configuration/MongoDbCucumberTestConfiguration.java +++ b/accession-commons-mongodb/src/test/java/uk/ac/ebi/ampt2d/test/configuration/MongoDbCucumberTestConfiguration.java @@ -25,6 +25,7 @@ import org.springframework.context.annotation.Scope; import org.springframework.data.mongodb.config.EnableMongoAuditing; import org.springframework.data.mongodb.repository.config.EnableMongoRepositories; +import uk.ac.ebi.ampt2d.commons.accession.core.AccessionSaveMode; import uk.ac.ebi.ampt2d.commons.accession.core.AccessioningService; import uk.ac.ebi.ampt2d.commons.accession.core.BasicAccessioningService; import uk.ac.ebi.ampt2d.commons.accession.core.DatabaseService; @@ -86,7 +87,8 @@ public AccessioningService testMongoDbAccessioningSer "id-service-" + o.getValue()), testMongoDbService(), TestModel::getValue, - new SHA1HashingFunction() + new SHA1HashingFunction(), + AccessionSaveMode.SAVE_ALL_THEN_RESOLVE ); } diff --git a/accession-commons-mongodb/src/test/java/uk/ac/ebi/ampt2d/test/configuration/MongoDbTestConfiguration.java b/accession-commons-mongodb/src/test/java/uk/ac/ebi/ampt2d/test/configuration/MongoDbTestConfiguration.java index 876eb2ba..c4559579 100644 --- a/accession-commons-mongodb/src/test/java/uk/ac/ebi/ampt2d/test/configuration/MongoDbTestConfiguration.java +++ b/accession-commons-mongodb/src/test/java/uk/ac/ebi/ampt2d/test/configuration/MongoDbTestConfiguration.java @@ -24,6 +24,7 @@ import org.springframework.context.annotation.Configuration; import org.springframework.data.mongodb.config.EnableMongoAuditing; import org.springframework.data.mongodb.repository.config.EnableMongoRepositories; +import uk.ac.ebi.ampt2d.commons.accession.core.AccessionSaveMode; import uk.ac.ebi.ampt2d.commons.accession.core.AccessioningService; import uk.ac.ebi.ampt2d.commons.accession.core.BasicAccessioningService; import uk.ac.ebi.ampt2d.commons.accession.core.DatabaseService; @@ -83,7 +84,8 @@ public AccessioningService testMongoDbAccessioningSer new SingleAccessionGenerator<>(o -> "id-" + o.getValue()), testMongoDbService(), testModel -> testModel.getValue(), - new SHA1HashingFunction() + new SHA1HashingFunction(), + AccessionSaveMode.SAVE_ALL_THEN_RESOLVE ); } diff --git a/accession-commons-monotonic-generator-jpa/src/test/java/uk/ac/ebi/ampt2d/commons/accession/core/BasicMonotonicAccessioningTest.java b/accession-commons-monotonic-generator-jpa/src/test/java/uk/ac/ebi/ampt2d/commons/accession/core/BasicMonotonicAccessioningTest.java index 0ca23ba9..8fc0a3d1 100644 --- a/accession-commons-monotonic-generator-jpa/src/test/java/uk/ac/ebi/ampt2d/commons/accession/core/BasicMonotonicAccessioningTest.java +++ b/accession-commons-monotonic-generator-jpa/src/test/java/uk/ac/ebi/ampt2d/commons/accession/core/BasicMonotonicAccessioningTest.java @@ -75,7 +75,8 @@ private AccessioningService getAccessioningService() { monotonicAccessionGenerator, databaseService, TestModel::getValue, - new SHA1HashingFunction() + new SHA1HashingFunction(), + AccessionSaveMode.SAVE_ALL_THEN_RESOLVE ); } diff --git a/accession-commons-monotonic-generator-jpa/src/test/java/uk/ac/ebi/ampt2d/commons/accession/core/BasicMonotonicAccessioningTestWithPreFiltering.java b/accession-commons-monotonic-generator-jpa/src/test/java/uk/ac/ebi/ampt2d/commons/accession/core/BasicMonotonicAccessioningTestWithPreFiltering.java new file mode 100644 index 00000000..1764f918 --- /dev/null +++ b/accession-commons-monotonic-generator-jpa/src/test/java/uk/ac/ebi/ampt2d/commons/accession/core/BasicMonotonicAccessioningTestWithPreFiltering.java @@ -0,0 +1,181 @@ +/* + * + * Copyright 2018 EMBL - European Bioinformatics Institute + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package uk.ac.ebi.ampt2d.commons.accession.core; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.orm.jpa.DataJpaTest; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.transaction.TestTransaction; +import uk.ac.ebi.ampt2d.commons.accession.core.exceptions.AccessionCouldNotBeGeneratedException; +import uk.ac.ebi.ampt2d.commons.accession.core.exceptions.AccessionDeprecatedException; +import uk.ac.ebi.ampt2d.commons.accession.core.exceptions.AccessionDoesNotExistException; +import uk.ac.ebi.ampt2d.commons.accession.core.exceptions.AccessionMergedException; +import uk.ac.ebi.ampt2d.commons.accession.core.models.AccessionWrapper; +import uk.ac.ebi.ampt2d.commons.accession.core.models.GetOrCreateAccessionWrapper; +import uk.ac.ebi.ampt2d.commons.accession.generators.monotonic.MonotonicAccessionGenerator; +import uk.ac.ebi.ampt2d.commons.accession.hashing.SHA1HashingFunction; +import uk.ac.ebi.ampt2d.test.configuration.TestMonotonicDatabaseServiceTestConfiguration; +import uk.ac.ebi.ampt2d.test.models.TestModel; +import uk.ac.ebi.ampt2d.test.persistence.TestMonotonicRepository; + +import java.util.Arrays; +import java.util.List; + +import static org.junit.Assert.assertEquals; + +@RunWith(SpringRunner.class) +@DataJpaTest +@ContextConfiguration(classes = {TestMonotonicDatabaseServiceTestConfiguration.class}) +public class BasicMonotonicAccessioningTestWithPreFiltering { + + private static String APPLICATION_INSTANCE_ID = "TEST_APPPLICATION_INSTANCE_ID"; + + @Autowired + private TestMonotonicRepository repository; + + @Autowired + private DatabaseService databaseService; + + @Autowired + private MonotonicAccessionGenerator monotonicAccessionGenerator; + + @Test + public void testCreateAccessions() throws AccessionCouldNotBeGeneratedException { + AccessioningService accessioningService = getAccessioningService(); + + List> accessions = accessioningService.getOrCreate( + Arrays.asList( + TestModel.of("service-test-1"), + TestModel.of("service-test-2"), + TestModel.of("service-test-3") + ), APPLICATION_INSTANCE_ID); + assertEquals(3, accessions.size()); + } + + private AccessioningService getAccessioningService() { + return new BasicAccessioningService<>( + monotonicAccessionGenerator, + databaseService, + TestModel::getValue, + new SHA1HashingFunction(), + AccessionSaveMode.PREFILTER_EXISTING + ); + } + + @Test + public void testGetOrCreateFiltersRepeated() throws AccessionCouldNotBeGeneratedException { + + AccessioningService accessioningService = getAccessioningService(); + + List> accessions = accessioningService.getOrCreate( + Arrays.asList( + TestModel.of("service-test-1"), + TestModel.of("service-test-2"), + TestModel.of("service-test-2"), + TestModel.of("service-test-3") + ), APPLICATION_INSTANCE_ID); + assertEquals(3, accessions.size()); + } + + @Test + public void testGetAccessions() throws AccessionCouldNotBeGeneratedException { + AccessioningService accessioningService = getAccessioningService(); + + List> accessions = accessioningService.get( + Arrays.asList( + TestModel.of("service-test-1"), + TestModel.of("service-test-2"), + TestModel.of("service-test-3") + )); + assertEquals(0, accessions.size()); + } + + @Test + public void testGetWithExistingEntries() throws AccessionCouldNotBeGeneratedException { + AccessioningService accessioningService = getAccessioningService(); + + List> accessions1 = accessioningService.getOrCreate( + Arrays.asList( + TestModel.of("service-test-3") + ), APPLICATION_INSTANCE_ID); + assertEquals(1, accessions1.size()); + + List> accessions2 = accessioningService.get( + Arrays.asList( + TestModel.of("service-test-1"), + TestModel.of("service-test-2"), + TestModel.of("service-test-3") + )); + assertEquals(1, accessions2.size()); + assertEquals("service-test-3", accessions2.get(0).getData().getValue()); + } + + @Test + public void testGetByAccessionsWithExistingEntries() throws AccessionCouldNotBeGeneratedException, + AccessionDoesNotExistException, AccessionMergedException, AccessionDeprecatedException { + AccessioningService accessioningService = getAccessioningService(); + + List> accessions1 = accessioningService.getOrCreate( + Arrays.asList( + TestModel.of("service-test-3") + ), APPLICATION_INSTANCE_ID); + assertEquals(1, accessions1.size()); + + AccessionWrapper accession2 = + accessioningService.getByAccession(accessions1.get(0).getAccession()); + assertEquals(accessions1.get(0).getAccession(), accession2.getAccession()); + } + + @Test + public void testGetOrCreateWithExistingEntries() throws AccessionCouldNotBeGeneratedException { + AccessioningService accessioningService = getAccessioningService(); + TestTransaction.flagForCommit(); + List> accessions1 = accessioningService.getOrCreate( + Arrays.asList( + TestModel.of("service-test-3") + ), APPLICATION_INSTANCE_ID); + assertEquals(1, accessions1.size()); + assertEquals(true, accessions1.get(0).isNewAccession()); + TestTransaction.end(); + + List> accessions2 = accessioningService.getOrCreate( + Arrays.asList( + TestModel.of("service-test-1"), + TestModel.of("service-test-2"), + TestModel.of("service-test-3") + ), APPLICATION_INSTANCE_ID); + assertEquals(3, accessions2.size()); + accessions2.stream().forEach(wrapper -> { + if (!wrapper.isNewAccession()) { + assertEquals("service-test-3",wrapper.getData().getValue()); + } + }); + assertEquals(2,accessions2.stream().filter(GetOrCreateAccessionWrapper::isNewAccession).count()); + + TestTransaction.start(); + for (AccessionWrapper accession : accessions2) { + repository.deleteById(accession.getHash()); + } + TestTransaction.flagForCommit(); + TestTransaction.end(); + } + +} diff --git a/accession-commons-monotonic-generator-jpa/src/test/java/uk/ac/ebi/ampt2d/commons/accession/core/BasicMonotonicAccessioningWithAlternateRangesTest.java b/accession-commons-monotonic-generator-jpa/src/test/java/uk/ac/ebi/ampt2d/commons/accession/core/BasicMonotonicAccessioningWithAlternateRangesTest.java index fbd9e35f..dd0c1a5b 100644 --- a/accession-commons-monotonic-generator-jpa/src/test/java/uk/ac/ebi/ampt2d/commons/accession/core/BasicMonotonicAccessioningWithAlternateRangesTest.java +++ b/accession-commons-monotonic-generator-jpa/src/test/java/uk/ac/ebi/ampt2d/commons/accession/core/BasicMonotonicAccessioningWithAlternateRangesTest.java @@ -226,7 +226,8 @@ private BasicAccessioningService getAccessioningService getGenerator(categoryId, instanceId), databaseService, TestModel::getValue, - new SHA1HashingFunction() + new SHA1HashingFunction(), + AccessionSaveMode.SAVE_ALL_THEN_RESOLVE ); } diff --git a/accession-commons-monotonic-generator-jpa/src/test/java/uk/ac/ebi/ampt2d/commons/accession/core/BasicMonotonicAccessioningWithAlternateRangesTestWithPreFiltering.java b/accession-commons-monotonic-generator-jpa/src/test/java/uk/ac/ebi/ampt2d/commons/accession/core/BasicMonotonicAccessioningWithAlternateRangesTestWithPreFiltering.java new file mode 100644 index 00000000..2f82f063 --- /dev/null +++ b/accession-commons-monotonic-generator-jpa/src/test/java/uk/ac/ebi/ampt2d/commons/accession/core/BasicMonotonicAccessioningWithAlternateRangesTestWithPreFiltering.java @@ -0,0 +1,238 @@ +/* + * + * Copyright 2018 EMBL - European Bioinformatics Institute + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package uk.ac.ebi.ampt2d.commons.accession.core; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.orm.jpa.DataJpaTest; +import org.springframework.test.annotation.DirtiesContext; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringRunner; +import uk.ac.ebi.ampt2d.commons.accession.block.initialization.BlockInitializationException; +import uk.ac.ebi.ampt2d.commons.accession.core.exceptions.AccessionCouldNotBeGeneratedException; +import uk.ac.ebi.ampt2d.commons.accession.core.models.AccessionWrapper; +import uk.ac.ebi.ampt2d.commons.accession.core.models.GetOrCreateAccessionWrapper; +import uk.ac.ebi.ampt2d.commons.accession.generators.monotonic.MonotonicAccessionGenerator; +import uk.ac.ebi.ampt2d.commons.accession.generators.monotonic.MonotonicRangePriorityQueue; +import uk.ac.ebi.ampt2d.commons.accession.hashing.SHA1HashingFunction; +import uk.ac.ebi.ampt2d.commons.accession.persistence.jpa.monotonic.entities.ContiguousIdBlock; +import uk.ac.ebi.ampt2d.commons.accession.persistence.jpa.monotonic.repositories.ContiguousIdBlockRepository; +import uk.ac.ebi.ampt2d.commons.accession.persistence.jpa.monotonic.service.ContiguousIdBlockService; +import uk.ac.ebi.ampt2d.commons.accession.service.BasicSpringDataRepositoryMonotonicDatabaseService; +import uk.ac.ebi.ampt2d.test.configuration.TestMonotonicDatabaseServiceTestConfiguration; +import uk.ac.ebi.ampt2d.test.models.TestModel; +import uk.ac.ebi.ampt2d.test.persistence.TestMonotonicEntity; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import java.util.stream.LongStream; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static uk.ac.ebi.ampt2d.commons.accession.util.ContiguousIdBlockUtil.getAllBlocksForCategoryId; +import static uk.ac.ebi.ampt2d.commons.accession.util.ContiguousIdBlockUtil.getAllUncompletedBlocksForCategoryId; +import static uk.ac.ebi.ampt2d.commons.accession.util.ContiguousIdBlockUtil.getUnreservedContiguousIdBlock; + +@RunWith(SpringRunner.class) +@DataJpaTest +@ContextConfiguration(classes = {TestMonotonicDatabaseServiceTestConfiguration.class}) +@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_EACH_TEST_METHOD) +public class BasicMonotonicAccessioningWithAlternateRangesTestWithPreFiltering { + + private static final String INSTANCE_ID = "test-instance"; + + @Autowired + private BasicSpringDataRepositoryMonotonicDatabaseService databaseService; + + @Autowired + private ContiguousIdBlockService contiguousIdBlockService; + + @Autowired + private ContiguousIdBlockRepository contiguousIdBlockRepository; + + @Test(expected = BlockInitializationException.class) + public void testUnknownCategory() throws AccessionCouldNotBeGeneratedException { + List> evaAccessions = + getAccessioningService("unknown-category", INSTANCE_ID) + .getOrCreate(getObjectsForAccessionsInRange(1, 10), INSTANCE_ID); + } + + @Test + public void testRecoverState() throws AccessionCouldNotBeGeneratedException { + String categoryId = "eva_2"; + String instanceId2 = "test-instance_2"; + + // create 3 un-complete contiguous id blocks of size 10 + // block-1 : (100 to 109), block-2 : (110 to 119), block-3 : (120 to 129) + List uncompletedBlocks = new ArrayList<>(); + uncompletedBlocks.add(getUnreservedContiguousIdBlock(categoryId, instanceId2, 100, 10)); + uncompletedBlocks.add(getUnreservedContiguousIdBlock(categoryId, instanceId2, 110, 10)); + uncompletedBlocks.add(getUnreservedContiguousIdBlock(categoryId, instanceId2, 120, 10)); + contiguousIdBlockService.save(uncompletedBlocks); + + assertEquals(3, getAllUncompletedBlocksForCategoryId(contiguousIdBlockRepository, categoryId).size()); + + // create and save accessions in db (100 to 124) - save 2 sets of same accessions with different hashes + List> accessionsSet1 = LongStream.range(100l, 125l) + .boxed() + .map(longAcc -> new AccessionWrapper<>(longAcc, "hash-1" + longAcc, TestModel.of("test-obj-1-" + longAcc))) + .collect(Collectors.toList()); + List> accessionsSet2 = LongStream.range(100l, 125l) + .boxed() + .map(longAcc -> new AccessionWrapper<>(longAcc, "hash-2" + longAcc, TestModel.of("test-obj-2-" + longAcc))) + .collect(Collectors.toList()); + databaseService.save(accessionsSet1); + databaseService.save(accessionsSet2); + + // run recover state + MonotonicAccessionGenerator generator = getGenerator(categoryId, instanceId2); + generator.generateAccessions(0, INSTANCE_ID); + + // As we have already saved accessions in db from 100 to 124, the status should be + // block-1 (100 to 109) : fully complete + // block-2 (110 to 119) : fully complete + // block-3 (120 to 124) : partially complete + assertEquals(1, getAllUncompletedBlocksForCategoryId(contiguousIdBlockRepository, categoryId).size()); + ContiguousIdBlock uncompletedBlock = getAllUncompletedBlocksForCategoryId(contiguousIdBlockRepository, categoryId).get(0); + assertEquals(120l, uncompletedBlock.getFirstValue()); + assertEquals(129l, uncompletedBlock.getLastValue()); + assertEquals(124l, uncompletedBlock.getLastCommitted()); + + MonotonicRangePriorityQueue availableRanges = generator.getAvailableRanges(); + assertEquals(1, availableRanges.size()); + assertEquals(125l, availableRanges.peek().getStart()); + assertEquals(129l, availableRanges.peek().getEnd()); + } + + @Test + public void testAlternateRangesWithDifferentGenerators() throws AccessionCouldNotBeGeneratedException { + /* blockStartValue= 0, blockSize= 10 , nextBlockInterval= 20 + the new blocks are interleaved or jumped for each 20 items accessioned + so the accessions will be in the range of 0-19,40-59,80-99 */ + String categoryId = "eva_2"; + String instanceId2 = "test-instance_2"; + BasicAccessioningService accService1 = getAccessioningService(categoryId, INSTANCE_ID); + List> evaAccessions = accService1.getOrCreate(getObjectsForAccessionsInRange(1, 9), INSTANCE_ID); + assertEquals(9, evaAccessions.size()); + assertEquals(0, evaAccessions.get(0).getAccession().longValue()); + assertEquals(8, evaAccessions.get(8).getAccession().longValue()); + //BlockSize of 10 was reserved but only 9 elements have been accessioned + assertEquals(1, getAllUncompletedBlocksForCategoryId(contiguousIdBlockRepository, categoryId).size()); + accService1.shutDownAccessioning(); + + //Get another service for same category + BasicAccessioningService accService2 = getAccessioningService(categoryId, INSTANCE_ID); + evaAccessions = accService2.getOrCreate(getObjectsForAccessionsInRange(11, 30), INSTANCE_ID); + assertEquals(20, evaAccessions.size()); + //Previous block ended here as only 9 elements were accessioned out of a blocksize of 10 + assertEquals(9, evaAccessions.get(0).getAccession().longValue()); + + //New Block still not interleaved or jumped as the interleave point is 20 + assertEquals(10, evaAccessions.get(1).getAccession().longValue()); + assertEquals(19, evaAccessions.get(10).getAccession().longValue()); + + //New Block interleaved as it reached interleave point 20 so jumped 20 places to 40 + assertEquals(40, evaAccessions.get(11).getAccession().longValue()); + assertEquals(48, evaAccessions.get(19).getAccession().longValue()); + //BlockSize if 10 was reserved but only 9 elements have been accessioned + assertEquals(1, getAllUncompletedBlocksForCategoryId(contiguousIdBlockRepository, categoryId).size()); + accService2.shutDownAccessioning(); + + //Get another service for same category but different Instance + BasicAccessioningService accService3 = getAccessioningService(categoryId, instanceId2); + evaAccessions = accService3.getOrCreate(getObjectsForAccessionsInRange(31, 39), INSTANCE_ID); + assertEquals(9, evaAccessions.size()); + //New Block from different instance have not jumped as still blocks are available before interleaving point + assertNotEquals(80, evaAccessions.get(0).getAccession().longValue()); + assertEquals(49, evaAccessions.get(0).getAccession().longValue()); + assertEquals(57, evaAccessions.get(8).getAccession().longValue()); + assertEquals(1, getAllUncompletedBlocksForCategoryId(contiguousIdBlockRepository, categoryId).size()); + accService3.shutDownAccessioning(); + + //Get previous uncompleted service from instance1 and create accessions + BasicAccessioningService accService4 = getAccessioningService(categoryId, INSTANCE_ID); + evaAccessions = accService4.getOrCreate(getObjectsForAccessionsInRange(40, 42), INSTANCE_ID); + assertEquals(3, evaAccessions.size()); + assertEquals(58, evaAccessions.get(0).getAccession().longValue()); //Block ended here + //New Block with 20 interval from last block made in instanceId2 + assertEquals(80, evaAccessions.get(2).getAccession().longValue()); + } + + @Test + public void testInitializeBlockManagerInMonotonicAccessionGenerator() throws AccessionCouldNotBeGeneratedException { + String categoryId = "eva_2"; + String instanceId2 = "test-instance_2"; + + ContiguousIdBlock block = getUnreservedContiguousIdBlock(categoryId, instanceId2, 0, 10); + contiguousIdBlockRepository.save(block); + + // assert block is not full and not reserved + List blockInDBList = getAllBlocksForCategoryId(contiguousIdBlockRepository, categoryId); + assertEquals(1, blockInDBList.size()); + List unreservedAndNotFullBlocks = blockInDBList.stream() + .filter(b -> b.isNotFull() && b.isNotReserved()) + .collect(Collectors.toList()); + assertEquals(1, unreservedAndNotFullBlocks.size()); + assertEquals(9, unreservedAndNotFullBlocks.get(0).getLastValue()); + assertEquals(-1, unreservedAndNotFullBlocks.get(0).getLastCommitted()); + assertEquals(false, unreservedAndNotFullBlocks.get(0).isReserved()); + + // this will run the recover state + MonotonicAccessionGenerator monotonicAccessionGenerator = getGenerator(categoryId, instanceId2); + monotonicAccessionGenerator.generateAccessions(0, INSTANCE_ID); + + // assert block gets reserved after recover state + blockInDBList = getAllBlocksForCategoryId(contiguousIdBlockRepository, categoryId); + assertEquals(1, blockInDBList.size()); + unreservedAndNotFullBlocks = blockInDBList.stream() + .filter(b -> b.isNotFull() && b.isNotReserved()) + .collect(Collectors.toList()); + assertEquals(0, unreservedAndNotFullBlocks.size()); + List reservedAndNotFullBlocks = blockInDBList.stream() + .filter(b -> b.isNotFull() && b.isReserved()) + .collect(Collectors.toList()); + assertEquals(1, reservedAndNotFullBlocks.size()); + assertEquals(9, reservedAndNotFullBlocks.get(0).getLastValue()); + assertEquals(-1, reservedAndNotFullBlocks.get(0).getLastCommitted()); + assertEquals(true, reservedAndNotFullBlocks.get(0).isReserved()); + } + + private List getObjectsForAccessionsInRange(int startRange, int endRange) { + return IntStream.range(startRange, endRange + 1).mapToObj(i -> TestModel.of("Test-" + i)).collect(Collectors + .toList()); + } + + private BasicAccessioningService getAccessioningService(String categoryId, + String instanceId) { + return new BasicAccessioningService<>( + getGenerator(categoryId, instanceId), + databaseService, + TestModel::getValue, + new SHA1HashingFunction(), + AccessionSaveMode.PREFILTER_EXISTING + ); + } + + private MonotonicAccessionGenerator getGenerator(String categoryId, String instanceId) { + return new MonotonicAccessionGenerator<>(categoryId, contiguousIdBlockService, databaseService); + } +} + diff --git a/accession-commons-monotonic-generator-jpa/src/test/java/uk/ac/ebi/ampt2d/commons/accession/core/BasicMonotonicAccessioningWithInitValuesTest.java b/accession-commons-monotonic-generator-jpa/src/test/java/uk/ac/ebi/ampt2d/commons/accession/core/BasicMonotonicAccessioningWithInitValuesTest.java index dc5ce692..b3708f49 100644 --- a/accession-commons-monotonic-generator-jpa/src/test/java/uk/ac/ebi/ampt2d/commons/accession/core/BasicMonotonicAccessioningWithInitValuesTest.java +++ b/accession-commons-monotonic-generator-jpa/src/test/java/uk/ac/ebi/ampt2d/commons/accession/core/BasicMonotonicAccessioningWithInitValuesTest.java @@ -75,7 +75,8 @@ private AccessioningService getAccessioningService() { monotonicAccessionGenerator, databaseService, TestModel::getValue, - new SHA1HashingFunction() + new SHA1HashingFunction(), + AccessionSaveMode.SAVE_ALL_THEN_RESOLVE ); } diff --git a/accession-commons-monotonic-generator-jpa/src/test/java/uk/ac/ebi/ampt2d/commons/accession/core/BasicMonotonicAccessioningWithInitValuesTestWithPreFiltering.java b/accession-commons-monotonic-generator-jpa/src/test/java/uk/ac/ebi/ampt2d/commons/accession/core/BasicMonotonicAccessioningWithInitValuesTestWithPreFiltering.java new file mode 100644 index 00000000..1ff37a69 --- /dev/null +++ b/accession-commons-monotonic-generator-jpa/src/test/java/uk/ac/ebi/ampt2d/commons/accession/core/BasicMonotonicAccessioningWithInitValuesTestWithPreFiltering.java @@ -0,0 +1,158 @@ +/* + * + * Copyright 2018 EMBL - European Bioinformatics Institute + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package uk.ac.ebi.ampt2d.commons.accession.core; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.orm.jpa.DataJpaTest; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.transaction.TestTransaction; +import uk.ac.ebi.ampt2d.commons.accession.core.exceptions.AccessionCouldNotBeGeneratedException; +import uk.ac.ebi.ampt2d.commons.accession.core.models.AccessionWrapper; +import uk.ac.ebi.ampt2d.commons.accession.core.models.GetOrCreateAccessionWrapper; +import uk.ac.ebi.ampt2d.commons.accession.generators.monotonic.MonotonicAccessionGenerator; +import uk.ac.ebi.ampt2d.commons.accession.hashing.SHA1HashingFunction; +import uk.ac.ebi.ampt2d.test.configuration.TestMonotonicDatabaseServiceTestConfiguration; +import uk.ac.ebi.ampt2d.test.models.TestModel; +import uk.ac.ebi.ampt2d.test.persistence.TestMonotonicEntity; +import uk.ac.ebi.ampt2d.test.persistence.TestMonotonicRepository; + +import java.util.Arrays; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +@RunWith(SpringRunner.class) +@DataJpaTest +@ContextConfiguration(classes = {TestMonotonicDatabaseServiceTestConfiguration.class}) +public class BasicMonotonicAccessioningWithInitValuesTestWithPreFiltering { + + private static String APPLICATION_INSTANCE_ID = "TEST_APPPLICATION_INSTANCE_ID"; + @Autowired + private TestMonotonicRepository repository; + + @Autowired + private DatabaseService databaseService; + + @Autowired + private MonotonicAccessionGenerator monotonicAccessionGenerator; + + @Test + public void testAccessionElements() throws AccessionCouldNotBeGeneratedException { + AccessioningService accessioningService = getAccessioningService(); + + List> accessions = accessioningService.getOrCreate( + Arrays.asList( + TestModel.of("service-test-1"), + TestModel.of("service-test-2"), + TestModel.of("service-test-3") + ), APPLICATION_INSTANCE_ID); + + assertEquals(3, accessions.size()); + accessions.stream().forEach(entry -> assertTrue(entry.getAccession() >= 100L)); + } + + private AccessioningService getAccessioningService() { + return new BasicAccessioningService<>( + monotonicAccessionGenerator, + databaseService, + TestModel::getValue, + new SHA1HashingFunction(), + AccessionSaveMode.PREFILTER_EXISTING + ); + } + + @Test + public void testGetOrCreateFiltersRepeated() throws AccessionCouldNotBeGeneratedException { + AccessioningService accessioningService = getAccessioningService(); + + List> accessions = accessioningService.getOrCreate( + Arrays.asList( + TestModel.of("service-test-1"), + TestModel.of("service-test-2"), + TestModel.of("service-test-2"), + TestModel.of("service-test-3") + ), APPLICATION_INSTANCE_ID); + assertEquals(3, accessions.size()); + accessions.stream().forEach(entry -> assertTrue(entry.getAccession() >= 100L)); + } + + @Test + public void testGetAccessions() throws AccessionCouldNotBeGeneratedException { + AccessioningService accessioningService = getAccessioningService(); + + List> accessions = accessioningService.get(Arrays.asList( + TestModel.of("service-test-1"), + TestModel.of("service-test-2"), + TestModel.of("service-test-3") + )); + assertEquals(0, accessions.size()); + } + + @Test + public void testGetWithExistingEntries() throws AccessionCouldNotBeGeneratedException { + repository.save(new TestMonotonicEntity( + 0L, + "85C4F271CBD3E11A9F8595854F755ADDFE2C0732", + 1, + "service-test-3")); + + AccessioningService accessioningService = getAccessioningService(); + + List> accessions = accessioningService.get(Arrays.asList( + TestModel.of("service-test-1"), + TestModel.of("service-test-2"), + TestModel.of("service-test-3") + )); + assertEquals(1, accessions.size()); + } + + @Test + public void testGetOrCreateWithExistingEntries() throws AccessionCouldNotBeGeneratedException { + TestTransaction.flagForCommit(); + repository.save(new TestMonotonicEntity( + 0L, + "85C4F271CBD3E11A9F8595854F755ADDFE2C0732", + 1, + "service-test-3")); + TestTransaction.end(); + + AccessioningService accessioningService = getAccessioningService(); + + List> accessions = accessioningService.getOrCreate( + Arrays.asList( + TestModel.of("service-test-1"), + TestModel.of("service-test-2"), + TestModel.of("service-test-3") + ), APPLICATION_INSTANCE_ID); + assertEquals(3, accessions.size()); + accessions.stream().forEach(entry -> + assertTrue(entry.getAccession() == 0L || entry.getAccession() >= 100L)); + + TestTransaction.start(); + for (AccessionWrapper accession : accessions) { + repository.deleteById(accession.getHash()); + } + TestTransaction.flagForCommit(); + TestTransaction.end(); + } + +} diff --git a/accession-commons-monotonic-generator-jpa/src/test/java/uk/ac/ebi/ampt2d/test/configuration/BasicRestControllerForDecoratedAccessionTestConfiguration.java b/accession-commons-monotonic-generator-jpa/src/test/java/uk/ac/ebi/ampt2d/test/configuration/BasicRestControllerForDecoratedAccessionTestConfiguration.java index d8d6d5fc..93b25551 100644 --- a/accession-commons-monotonic-generator-jpa/src/test/java/uk/ac/ebi/ampt2d/test/configuration/BasicRestControllerForDecoratedAccessionTestConfiguration.java +++ b/accession-commons-monotonic-generator-jpa/src/test/java/uk/ac/ebi/ampt2d/test/configuration/BasicRestControllerForDecoratedAccessionTestConfiguration.java @@ -28,6 +28,7 @@ import org.springframework.web.servlet.config.annotation.EnableWebMvc; import uk.ac.ebi.ampt2d.commons.accession.autoconfigure.EnableBasicRestControllerAdvice; import uk.ac.ebi.ampt2d.commons.accession.autoconfigure.EnableSpringDataContiguousIdService; +import uk.ac.ebi.ampt2d.commons.accession.core.AccessionSaveMode; import uk.ac.ebi.ampt2d.commons.accession.core.AccessioningService; import uk.ac.ebi.ampt2d.commons.accession.core.BasicAccessioningService; import uk.ac.ebi.ampt2d.commons.accession.core.DecoratedAccessioningService; @@ -110,7 +111,8 @@ public AccessioningService analysisAccessionService() { getDatabaseService()), getDatabaseService(), analysis -> analysis.getValue(), - new SHA1HashingFunction()); + new SHA1HashingFunction(), + AccessionSaveMode.SAVE_ALL_THEN_RESOLVE); } @Bean diff --git a/accession-commons-test/src/main/java/uk/ac/ebi/ampt2d/test/rest/MockTestAccessioningService.java b/accession-commons-test/src/main/java/uk/ac/ebi/ampt2d/test/rest/MockTestAccessioningService.java index 6825c0d5..6b04b3c7 100644 --- a/accession-commons-test/src/main/java/uk/ac/ebi/ampt2d/test/rest/MockTestAccessioningService.java +++ b/accession-commons-test/src/main/java/uk/ac/ebi/ampt2d/test/rest/MockTestAccessioningService.java @@ -17,6 +17,7 @@ */ package uk.ac.ebi.ampt2d.test.rest; +import uk.ac.ebi.ampt2d.commons.accession.core.AccessionSaveMode; import uk.ac.ebi.ampt2d.commons.accession.core.BasicAccessioningService; import uk.ac.ebi.ampt2d.commons.accession.core.DatabaseService; import uk.ac.ebi.ampt2d.test.models.TestModel; @@ -32,7 +33,8 @@ public MockTestAccessioningService(DatabaseService da new MockTestAccessionGenerator(), databaseService, testModel -> testModel.getValue(), - s -> "hash-" + s + s -> "hash-" + s, + AccessionSaveMode.SAVE_ALL_THEN_RESOLVE ); }