From 1862b58f3f088089e997fb8aa0a693ba24e6c3c7 Mon Sep 17 00:00:00 2001 From: nkumar2 Date: Mon, 18 May 2026 13:51:48 +0100 Subject: [PATCH 01/10] add java21 branch for testing --- .github/workflows/run-tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index e82393b3d..6bd664e22 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -2,10 +2,10 @@ name: Run tests on: push: - branches: [ master ] + branches: [ master, java21 ] pull_request: # Temporarily run tests on the feature branch until merged to master - branches: [ master, feature/ws-contig-alias-integration ] + branches: [ master, java21, feature/ws-contig-alias-integration ] jobs: build: From a520f5f9c1b0c6efde9b3cfac85ce57b16b13e6c Mon Sep 17 00:00:00 2001 From: nkumar2 Date: Mon, 18 May 2026 13:51:48 +0100 Subject: [PATCH 02/10] add java21 branch for testing --- .github/workflows/run-tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index e82393b3d..6bd664e22 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -2,10 +2,10 @@ name: Run tests on: push: - branches: [ master ] + branches: [ master, java21 ] pull_request: # Temporarily run tests on the feature branch until merged to master - branches: [ master, feature/ws-contig-alias-integration ] + branches: [ master, java21, feature/ws-contig-alias-integration ] jobs: build: From b51b10836c09aaae41a9846d7066bce8480d8949 Mon Sep 17 00:00:00 2001 From: nitin-ebi <79518737+nitin-ebi@users.noreply.github.com> Date: Mon, 1 Jun 2026 19:51:31 +0100 Subject: [PATCH 03/10] Upgrade project - java 21 - spring 3.4 (#501) --- .github/workflows/run-tests.yml | 42 +-- .gitlab-ci.yml | 2 +- eva-accession-clustering/pom.xml | 54 +++- .../batch/io/BackPropagatedRSWriter.java | 26 +- .../clustering/batch/io/ClusteringWriter.java | 127 ++++---- .../batch/io/ListOfListItemWriter.java | 7 +- .../clustering/batch/io/RSMergeWriter.java | 134 ++++---- .../clustering/batch/io/RSSplitWriter.java | 68 ++-- .../clustering/batch/io/SSSplitWriter.java | 85 +++-- ...getSSReaderForSplitOrMergedBackPropRS.java | 15 +- .../batch/io/qc/DuplicateRSAccQCWriter.java | 7 +- .../batch/io/qc/ExtraneousRSReporter.java | 22 +- .../batch/io/qc/MissingCveReporter.java | 12 +- .../io/qc/PendingMergeSplitReporter.java | 37 +-- .../clustering/batch/io/qc/RSReader.java | 11 +- .../clustering/batch/io/qc/SSReader.java | 12 +- ...iantToSubmittedVariantEntityProcessor.java | 5 +- .../batch/RunnerConfiguration.java | 12 - ...AndSplitCandidatesReaderConfiguration.java | 30 +- .../jobs/BackPropagateRSJobConfiguration.java | 16 +- ...erUnclusteredVariantsJobConfiguration.java | 23 +- .../ClusteringFromMongoJobConfiguration.java | 67 ++-- ...emappedVariantsWithRSJobConfiguration.java | 13 +- .../RSAccessionRecoveryJobConfiguration.java | 7 +- .../jobs/StudyClusteringJobConfiguration.java | 17 +- .../jobs/qc/ClusteringQCJobConfiguration.java | 32 +- .../qc/DuplicateRSAccQCJobConfiguration.java | 7 +- ...ewClusteredVariantsQCJobConfiguration.java | 26 +- ...AccessioningShutdownStepConfiguration.java | 10 +- .../ClusteringFromMongoStepConfiguration.java | 128 ++++---- .../RSAccessionRecoveryStepConfiguration.java | 11 +- .../qc/DuplicateRSAccQCStepConfiguration.java | 10 +- .../metric/ClusteringMetricCompute.java | 17 +- .../parameters/InputParameters.java | 3 +- .../runner/ClusteringCommandLineRunner.java | 19 +- .../batch/io/BackPropagatedRSWriterTest.java | 62 ++-- .../ClusteredVariantSplittingPolicyTest.java | 10 +- .../io/ClusteringMongoReaderRetryTest.java | 20 +- .../batch/io/ClusteringMongoReaderTest.java | 54 ++-- .../RSMergeAndSplitCandidatesReaderTest.java | 144 ++++----- .../batch/io/RSMergeWriterTest.java | 204 ++++++------ .../batch/io/RSSplitWriterTest.java | 165 ++++------ .../batch/io/SSSplitWriterTest.java | 106 +++---- .../io/StudyClusteringMongoReaderTest.java | 54 ++-- .../ClusteringAssertions.java | 2 +- .../IssueAccessionClusteringWriterTest.java | 125 ++++---- .../MergeAccessionClusteringWriterTest.java | 145 ++++----- .../RemappedVariantsClusteringWriterTest.java | 233 +++++++------- .../ReuseAccessionClusteringWriterTest.java | 71 ++--- ...ToSubmittedVariantEntityProcessorTest.java | 10 +- ...ClusteringVariantJobConfigurationTest.java | 92 +++--- .../batch/jobs/RSAccessionRecoveryTest.java | 35 +- ...DuplicateRSAccQCStepConfigurationTest.java | 82 ++--- ...usteredVariantsQCJobConfigurationTest.java | 55 ++-- ...lusteringVariantStepConfigurationTest.java | 66 ++-- .../ClusteringCommandLineRunnerTest.java | 143 ++++----- .../clustering/test/VariantAssertions.java | 2 +- .../BatchJobRepositoryTestConfiguration.java | 22 ++ .../configuration/BatchTestConfiguration.java | 117 +++---- .../configuration/MongoTestConfiguration.java | 72 ++++- .../RSAccessionRecoveryTestConfiguration.java | 46 +-- .../test/rule/FixSpringMongoDbRule.java | 40 --- .../resources/backpropagation-test.properties | 3 - .../clustering-issuance-test.properties | 3 - .../clustering-pipeline-test.properties | 4 +- .../resources/clustering-qc-test.properties | 3 - .../clustering-writer-test.properties | 3 - .../duplicate-rs-acc-qc-test.properties | 3 - .../resources/merge-split-test.properties | 3 - .../rs-accession-recovery.properties | 10 +- eva-accession-core/pom.xml | 20 +- .../io/ClusteredVariantDeprecationWriter.java | 50 +-- .../DbsnpClusteredVariantOperationWriter.java | 12 +- .../batch/io/DbsnpClusteredVariantWriter.java | 15 +- .../io/SubmittedVariantDeprecationWriter.java | 52 +-- .../listeners/GenericProgressListener.java | 9 +- .../policies/IllegalStartSkipPolicy.java | 2 +- .../configuration/MongoClientCreator.java | 37 ++- .../human/HumanMongoConfiguration.java | 29 +- .../nonhuman/MongoConfiguration.java | 53 ++-- ...ittedVariantAccessioningConfiguration.java | 18 +- .../core/model/ClusteredVariant.java | 4 +- .../DbsnpSubmittedVariantInactiveEntity.java | 1 - .../core/runner/CommandLineRunnerUtils.java | 31 +- .../DbsnpSubmittedVariantInactiveService.java | 3 +- .../eva/SubmittedVariantInactiveService.java | 3 +- .../core/EVAObjectModelUtilsTest.java | 9 +- .../batch/io/AccessionedVcfFactoryTest.java | 15 +- ...ClusteredVariantDeprecationWriterTest.java | 83 ++--- .../io/DbsnpClusteredVariantWriterTest.java | 65 ++-- .../batch/io/FastaSequenceReaderTest.java | 68 ++-- ...SubmittedVariantDeprecationWriterTest.java | 88 ++---- .../policies/IllegalStartSkipPolicyTest.java | 10 +- .../core/contig/ContigMappingTest.java | 14 +- ...ClusteredVariantEntityIntegrationTest.java | 41 +-- ...leaseRecordSubmittedVariantEntityTest.java | 4 +- ...SubmittedVariantEntityIntegrationTest.java | 46 +-- .../model/eva/SubmittedVariantEntityTest.java | 25 +- .../core/mongoTemplate/MongoTemplateTest.java | 23 +- ...eredVariantAccessioningRepositoryTest.java | 63 ++-- ...ttedVariantAccessioningRepositoryTest.java | 81 +++-- ...usteredVariantAccessioningServiceTest.java | 60 ++-- ...usteredVariantAccessioningServiceTest.java | 49 ++- ...bmittedVariantAccessioningServiceTest.java | 235 +++++++------- ...riantMonotonicAccessioningServiceTest.java | 37 ++- .../ClusteredVariantSummaryFunctionTest.java | 10 +- .../SubmittedVariantSummaryFunctionTest.java | 10 +- .../configuration/DbsnpTestDataSource.java | 1 - .../configuration/JPATestConfiguration.java | 41 +++ .../human/MongoHumanTestConfiguration.java | 69 +++- .../nonhuman/MongoTestConfiguration.java | 70 ++++ .../core/test/rule/FixSpringMongoDbRule.java | 41 --- .../core/utils/MongoTestContainerHelper.java | 24 ++ .../core/utils/MongoTestDataLoader.java | 106 +++++++ .../utils/PipelineTemporaryFolderUtil.java | 107 +++++++ .../properties/rs-accession-test.properties | 3 - .../properties/ss-accession-test.properties | 7 - .../properties/ss-deprecation-test.properties | 7 - .../properties/test-model.properties | 3 - .../test-variants-writer.properties | 4 - eva-accession-deprecate/pom.xml | 30 +- ...ySubmittedVariantsReaderConfiguration.java | 3 +- ...tudySubmittedVariantsJobConfiguration.java | 26 +- ...ittedVariantsFromFileJobConfiguration.java | 19 +- .../listeners/ListenerConfiguration.java | 5 +- ...udySubmittedVariantsStepConfiguration.java | 12 +- ...ttedVariantsFromFileStepConfiguration.java | 13 +- .../deprecate/MongoTestDatabaseSetup.java | 4 +- .../io/StudySubmittedVariantsReaderTest.java | 49 +-- ...SubmittedVariantsJobConfigurationTest.java | 71 ++--- ...dVariantsFromFileJobConfigurationTest.java | 59 ++-- ...ubmittedVariantsStepConfigurationTest.java | 50 +-- ...VariantsFromFileStepConfigurationTest.java | 49 +-- .../BatchJobRepositoryTestConfiguration.java | 22 ++ .../configuration/BatchTestConfiguration.java | 41 ++- .../configuration/MongoTestConfiguration.java | 70 ++++ .../test/rule/FixSpringMongoDbRule.java | 42 --- ...bmitted-variants-from-file-test.properties | 7 - .../study-submitted-variants-test.properties | 7 - eva-accession-import-dbsnp2/pom.xml | 26 +- .../io/DbsnpJsonClusteredVariantsWriter.java | 23 +- ...DbsnpJsonVariantsStepProgressListener.java | 6 +- .../ContigToGenbankReplacerProcessor.java | 5 +- .../configuration/RunnerConfiguration.java | 13 - ...tDbsnpJsonVariantsWriterConfiguration.java | 4 +- ...portDbsnpJsonVariantsJobConfiguration.java | 7 +- ...ortDbsnpJsonVariantsStepConfiguration.java | 24 +- ...tVariantsJobLauncherCommandLineRunner.java | 22 +- .../DbsnpJsonClusteredVariantsWriterTest.java | 129 ++++---- .../batch/io/DbsnpJsonItemReaderTest.java | 21 +- .../ContigToGenbankReplacerProcessorTest.java | 67 ++-- ...onNodeToClusteredVariantProcessorTest.java | 21 +- ...DbsnpJsonVariantsJobConfigurationTest.java | 23 +- ...bsnpJsonVariantsStepConfigurationTest.java | 26 +- ...iantsJobLauncherCommandLineRunnerTest.java | 73 +++-- .../BatchJobRepositoryTestConfiguration.java | 22 ++ .../dbsnp2/test/BatchTestConfiguration.java | 37 ++- .../dbsnp2/test/MongoTestConfiguration.java | 102 ++++++ .../src/test/resources/application.properties | 3 - .../test-dbsnp-json-import.properties | 0 eva-accession-pipeline/pom.xml | 33 +- .../batch/io/AccessionReportWriter.java | 106 +++---- .../pipeline/batch/io/AccessionWriter.java | 51 +-- .../batch/io/DuplicateSSAccQCWriter.java | 7 +- .../policies/InvalidVariantSkipPolicy.java | 2 +- .../configuration/RunnerConfiguration.java | 13 - .../io/AccessionWriterConfiguration.java | 9 +- .../batch/io/VcfReaderConfiguration.java | 5 +- .../DuplicateSSAccQCJobConfiguration.java | 7 +- .../QCSubsnpAccessionsJobConfiguration.java | 7 +- .../SSAccessionRecoveryJobConfiguration.java | 7 +- .../SubsnpAccessionsJobConfiguration.java | 7 +- .../VariantProcessorConfiguration.java | 3 +- ...AccessioningShutdownStepConfiguration.java | 10 +- .../steps/BuildReportStepConfiguration.java | 16 +- .../DuplicateSSAccQCStepConfiguration.java | 10 +- .../QCSubsnpAccessionsStepConfiguration.java | 10 +- .../SSAccessionRecoveryStepConfiguration.java | 10 +- .../SubsnpAccessionsStepConfiguration.java | 10 +- .../metric/AccessioningMetricCompute.java | 16 +- ...AccessionJobLauncherCommandLineRunner.java | 20 +- .../batch/io/AccessionReportWriterTest.java | 102 +++--- .../batch/io/AccessionWriterTest.java | 111 ++++--- .../InvalidVariantSkipPolicyTest.java | 10 +- .../ContigToGenbankReplacerProcessorTest.java | 10 +- ...xcludeStructuralVariantsProcessorTest.java | 12 +- .../processors/VariantConverterTest.java | 67 ++-- .../buildReport/BuildReportTaskletTest.java | 29 +- .../AccessionWrapperComparatorTest.java | 21 +- .../reportCheck/ReportCheckTaskletTest.java | 7 +- ...fSubsnpAccessionsJobConfigurationTest.java | 26 +- ...eSubsnpAccessionsJobConfigurationTest.java | 28 +- ...reateSubsnpAccessionsRecoverStateTest.java | 45 ++- ...nsRecoveringStateJobConfigurationTest.java | 48 ++- .../DuplicateSSAccQCJobConfigurationTest.java | 55 ++-- ...CSubsnpAccessionsJobConfigurationTest.java | 18 +- ...SubsnpAccessionsStepConfigurationTest.java | 26 +- ...SubsnpAccessionsStepConfigurationTest.java | 22 +- ...ssionJobLauncherCommandLineRunnerTest.java | 50 +-- .../runner/JobFailureBlocksReleasedTest.java | 34 +- .../pipeline/runner/RestartFailedJobTest.java | 47 ++- .../runner/SSAccessionRecoveryTest.java | 22 +- .../BatchJobRepositoryTestConfiguration.java | 22 ++ .../pipeline/test/BatchTestConfiguration.java | 50 ++- .../pipeline/test/FixSpringMongoDbRule.java | 41 --- .../pipeline/test/MongoTestConfiguration.java | 102 ++++++ .../SSAccessionRecoveryTestConfiguration.java | 20 +- ...ccession-pipeline-interval-test.properties | 3 - ...ion-pipeline-recover-state-test.properties | 11 +- ...accession-pipeline-recover-test.properties | 11 +- .../accession-pipeline-test.properties | 4 +- .../duplicate-ss-acc-qc-test.properties | 3 - .../qc-accession-pipeline-test.properties | 7 +- .../ss-accession-recovery.properties | 11 +- eva-accession-release/pom.xml | 29 +- .../assembly/AssemblyNameRetriever.java | 17 +- .../io/VariantMongoAggregationReader.java | 17 +- .../active/AccessionedVariantMongoReader.java | 2 +- .../batch/io/active/VariantContextWriter.java | 39 ++- .../batch/io/contig/ContigMongoReader.java | 32 +- .../release/batch/io/contig/ContigWriter.java | 5 +- .../DeprecatedVariantAccessionWriter.java | 7 +- .../DeprecatedVariantMongoReader.java | 20 +- .../io/merged/MergedVariantContextWriter.java | 3 +- .../io/merged/MergedVariantMongoReader.java | 56 ++-- ...snpMergedDeprecatedVariantMongoReader.java | 10 +- ...EvaMergedDeprecatedVariantMongoReader.java | 10 +- ...ergedDeprecatedVariantAccessionWriter.java | 4 +- .../MergedDeprecatedVariantMongoReader.java | 36 +-- .../MultimapVariantContextWriter.java | 4 +- .../multimap/MultimapVariantMongoReader.java | 10 +- .../listeners/ExcludeVariantsListener.java | 2 +- .../ExcludeInvalidVariantsProcessor.java | 7 +- .../processors/NamedVariantProcessor.java | 32 +- .../VariantToVariantContextProcessor.java | 11 +- .../configuration/RunnerConfiguration.java | 21 -- ...ccessionReleaseFromDBJobConfiguration.java | 7 +- .../DumpRSAccessionsJobConfiguration.java | 11 +- ...ccessionReleaseFromDBJobConfiguration.java | 7 +- .../ReleaseProcessorConfiguration.java | 5 +- ...cessionReleaseFromDBStepConfiguration.java | 10 +- .../DumpRSAccessionsStepConfiguration.java | 18 +- ...cessionReleaseFromDBStepConfiguration.java | 10 +- .../assembly/AssemblyNameRetrieverTest.java | 41 ++- .../AccessionedVariantMongoReaderTest.java | 225 ++++++------- .../io/active/VariantContextWriterTest.java | 143 +++++---- .../io/contig/ContigMongoReaderTest.java | 58 ++-- .../batch/io/contig/ContigWriterTest.java | 62 ++-- .../DeprecatedVariantAccessionWriterTest.java | 21 +- .../DeprecatedVariantMongoReaderTest.java | 74 ++--- .../merged/MergedVariantMongoReaderTest.java | 250 +++++++-------- ...dDeprecatedVariantAccessionWriterTest.java | 24 +- ...ergedDeprecatedVariantMongoReaderTest.java | 60 ++-- ...ontextNucleotideAdditionProcessorTest.java | 44 +-- .../ExcludeInvalidVariantsProcessorTest.java | 18 +- .../processors/NamedVariantProcessorTest.java | 17 +- .../VariantToVariantContextProcessorTest.java | 65 ++-- .../batch/steps/DumpRSAccessionsTest.java | 47 +-- .../steps/RSAccessionsReleaseFromDBTest.java | 47 +-- .../BatchJobRepositoryTestConfiguration.java | 22 ++ .../configuration/MongoTestConfiguration.java | 70 ++++ .../ReleaseFromDBTestConfiguration.java | 66 ++-- .../test/rule/FixSpringMongoDbRule.java | 41 --- .../src/test/resources/application.properties | 3 - .../dump-rs-accession-test.properties | 3 - ...cremental-release-pipeline-test.properties | 3 - .../release-pipeline-test.properties | 3 - eva-accession-ws/pom.xml | 38 ++- .../accession/ws/EvaAccessionApplication.java | 2 +- .../ApplicationConfiguration.java | 36 +-- .../configuration/SwaggerConfiguration.java | 52 +-- .../rest/ClusteredVariantsRestController.java | 85 +++-- .../ws/rest/EvaControllerAdvice.java | 2 +- .../rest/SubmittedVariantsRestController.java | 74 +++-- .../ClusteredVariantsBeaconService.java | 13 +- .../ClusteredVariantHistoryEndPointTest.java | 84 ++--- .../ClusteredVariantsRestControllerTest.java | 299 +++++++++--------- .../SubmittedVariantsRestControllerTest.java | 166 +++++----- .../ws/test/MongoTestConfiguration.java | 105 ++++++ .../accession/ws/test/TestConfiguration.java | 5 +- .../ws/test/rule/FixSpringMongoDbRule.java | 41 --- .../resources/accession-ws-test.properties | 3 - eva-remapping-get-source/pom.xml | 25 +- .../io/DbsnpSubmittedVariantMongoReader.java | 5 +- .../io/EvaSubmittedVariantMongoReader.java | 3 +- .../source/batch/io/VariantContextWriter.java | 14 +- ...ittedVariantToVariantContextProcessor.java | 3 +- .../configuration/RunnerConfiguration.java | 13 - .../io/VariantContextWriterConfiguration.java | 3 +- ...portSubmittedVariantsJobConfiguration.java | 18 +- .../listeners/ListenersConfiguration.java | 5 +- ...ortSubmittedVariantsStepConfiguration.java | 21 +- .../source/parameters/InputParameters.java | 5 +- ...RemappingJobLauncherCommandLineRunner.java | 22 +- .../DbsnpSubmittedVariantMongoReaderTest.java | 47 ++- .../EvaSubmittedVariantMongoReaderTest.java | 48 ++- .../batch/io/VariantContextWriterTest.java | 52 ++- ...ontextNucleotideAdditionProcessorTest.java | 44 +-- ...eAllelesMismatchVariantsProcessorTest.java | 12 +- .../ExcludeMultimapVariantsProcessorTest.java | 16 +- .../VariantToVariantContextProcessorTest.java | 21 +- ...SubmittedVariantsJobConfigurationTest.java | 88 +++--- ...ppingJobLauncherCommandLineRunnerTest.java | 69 ++-- ...cherCommandLineRunnerWithProjectsTest.java | 66 ++-- .../BatchJobRepositoryTestConfiguration.java | 22 ++ .../configuration/BatchTestConfiguration.java | 30 +- .../configuration/MongoTestConfiguration.java | 70 ++++ .../src/test/resources/application.properties | 1 - .../test/resources/with-projects.properties | 1 - eva-remapping-ingest/pom.xml | 31 +- .../io/RemappedSubmittedVariantsWriter.java | 90 +++--- .../configuration/RunnerConfiguration.java | 13 - .../batch/io/VcfReaderConfiguration.java | 3 +- ...mappedVariantsFromVcfJobConfiguration.java | 15 +- ...ngestRemappedFromVcfStepConfiguration.java | 21 +- ...oreRemappingMetadataStepConfiguration.java | 15 +- .../IngestRemappedCommandLineRunner.java | 20 +- .../RemappedSubmittedVariantsWriterTest.java | 77 ++--- ...edVariantsFromVcfJobConfigurationTest.java | 59 ++-- ...tRemappedFromVcfStepConfigurationTest.java | 61 ++-- ...ppedFromVcfStepDbsnpConfigurationTest.java | 62 ++-- .../BatchJobRepositoryTestConfiguration.java | 22 ++ .../configuration/BatchTestConfiguration.java | 30 +- .../configuration/MongoTestConfiguration.java | 70 ++++ .../test/rule/FixSpringMongoDbRule.java | 38 --- .../ingest-remapped-variants-dbsnp.properties | 3 - .../ingest-remapped-variants.properties | 3 - pom.xml | 168 +++++++--- 328 files changed, 6328 insertions(+), 5899 deletions(-) create mode 100644 eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/test/configuration/BatchJobRepositoryTestConfiguration.java delete mode 100644 eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/test/rule/FixSpringMongoDbRule.java create mode 100644 eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/test/configuration/JPATestConfiguration.java delete mode 100644 eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/test/rule/FixSpringMongoDbRule.java create mode 100644 eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/utils/MongoTestContainerHelper.java create mode 100644 eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/utils/MongoTestDataLoader.java create mode 100644 eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/utils/PipelineTemporaryFolderUtil.java create mode 100644 eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/test/configuration/BatchJobRepositoryTestConfiguration.java delete mode 100644 eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/test/rule/FixSpringMongoDbRule.java create mode 100644 eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/test/BatchJobRepositoryTestConfiguration.java create mode 100644 eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/test/MongoTestConfiguration.java delete mode 100644 eva-accession-import-dbsnp2/src/test/resources/test-dbsnp-json-import.properties create mode 100644 eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/test/BatchJobRepositoryTestConfiguration.java delete mode 100644 eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/test/FixSpringMongoDbRule.java create mode 100644 eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/test/MongoTestConfiguration.java create mode 100644 eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/test/configuration/BatchJobRepositoryTestConfiguration.java delete mode 100644 eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/test/rule/FixSpringMongoDbRule.java create mode 100644 eva-accession-ws/src/test/java/uk/ac/ebi/eva/accession/ws/test/MongoTestConfiguration.java delete mode 100644 eva-accession-ws/src/test/java/uk/ac/ebi/eva/accession/ws/test/rule/FixSpringMongoDbRule.java create mode 100644 eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/test/configuration/BatchJobRepositoryTestConfiguration.java create mode 100644 eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/test/configuration/BatchJobRepositoryTestConfiguration.java delete mode 100644 eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/test/rule/FixSpringMongoDbRule.java diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 6bd664e22..ed97e97fd 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -4,42 +4,26 @@ on: push: branches: [ master, java21 ] pull_request: - # Temporarily run tests on the feature branch until merged to master - branches: [ master, java21, feature/ws-contig-alias-integration ] + branches: [ master, java21 ] jobs: build: runs-on: ubuntu-latest + strategy: matrix: - # Including "ubuntu2204" in the version is a temporary fix to the direct download URL while we transition - # between 4 and 6 - mongodb-version: [4.0.18, ubuntu2204-6.0.6] - java-version: [1.8] - python-version: [3.8] + java-version: [21] steps: - - uses: actions/checkout@v2 - - name: Setup Java JDK - uses: actions/setup-java@v1.4.3 - # The Java version to make available on the path. Takes a whole or semver Java version, or 1.x syntax (e.g. 1.8 => Java 8.x). Early access versions can be specified in the form of e.g. 14-ea, 14.0.0-ea, or 14.0.0-ea.28 - with: - java-version: ${{ matrix.java-version }} - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Java 21 + uses: actions/setup-java@v4 with: - python-version: ${{ matrix.python-version }} - - name: Install Python dependencies - run: | - python -m pip install --upgrade pip - pip install numpy pandas psycopg2-binary pgpasslib - - name: Install and Start MongoDB - run: | - wget https://fastdl.mongodb.org/linux/mongodb-linux-x86_64-${{ matrix.mongodb-version }}.tgz - tar xfz mongodb-linux-x86_64-${{ matrix.mongodb-version }}.tgz - export PATH=`pwd`/mongodb-linux-x86_64-${{ matrix.mongodb-version }}/bin:$PATH - mkdir -p data/db - mongod --dbpath=data/db & - mongod --version + distribution: temurin + java-version: ${{ matrix.java-version }} + cache: maven + - name: Run Maven tests - run: mvn test + run: mvn -B clean test \ No newline at end of file diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 69b6eda0f..02f0123f0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -28,7 +28,7 @@ test: - mongo:4.0.18 script: # Gitlab exposes services under their own hostnames. So test host should be "mongo" instead of "localhost". - - mvn clean test --projects 'eva-accession-core,eva-accession-deprecate,eva-accession-ws,eva-remapping-get-source,eva-accession-clustering,eva-accession-pipeline,eva-accession-release,eva-remapping-ingest' -Deva.mongo.host.test=mongo + - mvn clean test --projects 'eva-accession-core,eva-accession-deprecate,eva-accession-ws,eva-remapping-get-source,eva-accession-clustering,eva-accession-pipeline,eva-accession-release,eva-remapping-ingest' environment: name: test-env only: diff --git a/eva-accession-clustering/pom.xml b/eva-accession-clustering/pom.xml index c0bb9c559..53ac31e80 100644 --- a/eva-accession-clustering/pom.xml +++ b/eva-accession-clustering/pom.xml @@ -2,12 +2,13 @@ + 4.0.0 + eva-accession uk.ac.ebi.eva - 0.6.57-SNAPSHOT + ${revision} - 4.0.0 eva-accession-clustering jar @@ -31,9 +32,8 @@ - com.google.guava - guava - 33.4.0-jre + com.google.code.gson + gson @@ -41,6 +41,14 @@ xstream + + uk.ac.ebi.eva + eva-accession-core + ${revision} + test-jar + test + + org.springframework.batch spring-batch-test @@ -48,8 +56,24 @@ - com.lordofthejars - nosqlunit-mongodb + org.springframework.boot + spring-boot-starter-test + test + + + + org.springframework.boot + spring-boot-testcontainers + test + + + org.testcontainers + mongodb + test + + + org.testcontainers + junit-jupiter test @@ -76,9 +100,19 @@ - src/test/resources/ + src/test/resources + + **/*.gz + true + + src/test/resources + + **/*.gz + + false + @@ -90,8 +124,4 @@ - - localhost - - diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/io/BackPropagatedRSWriter.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/io/BackPropagatedRSWriter.java index 32ddc983c..0f9b8a1e1 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/io/BackPropagatedRSWriter.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/io/BackPropagatedRSWriter.java @@ -16,13 +16,13 @@ package uk.ac.ebi.eva.accession.clustering.batch.io; import com.mongodb.MongoBulkWriteException; - +import jakarta.annotation.Nonnull; +import org.springframework.batch.item.Chunk; import org.springframework.batch.item.ItemWriter; import org.springframework.data.mongodb.core.BulkOperations; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.data.mongodb.core.query.Query; import org.springframework.data.mongodb.core.query.Update; - import uk.ac.ebi.ampt2d.commons.accession.core.exceptions.AccessionCouldNotBeGeneratedException; import uk.ac.ebi.ampt2d.commons.accession.core.models.EventType; import uk.ac.ebi.ampt2d.commons.accession.persistence.mongodb.document.AccessionedDocument; @@ -36,7 +36,6 @@ import uk.ac.ebi.eva.accession.core.service.nonhuman.SubmittedVariantAccessioningService; import uk.ac.ebi.eva.metrics.metric.MetricCompute; -import javax.annotation.Nonnull; import java.util.Collections; import java.util.Comparator; import java.util.List; @@ -74,11 +73,10 @@ public BackPropagatedRSWriter(String remappedAssembly, ClusteringWriter clusteri } @Override - public void write( - @Nonnull List submittedVariantEntitiesInOriginalAssembly) + public void write(@Nonnull Chunk submittedVariantEntitiesInOriginalAssembly) throws MongoBulkWriteException, AccessionCouldNotBeGeneratedException { List ssToLookupInRemappedAssembly = - submittedVariantEntitiesInOriginalAssembly + submittedVariantEntitiesInOriginalAssembly.getItems() .stream() // Some dbSNP imported variants might have been explicitly de-clustered // because the REF/ALT allele data provided by dbSNP was internally inconsistent @@ -89,15 +87,15 @@ public void write( .collect(Collectors.toList()); List ssIDsToLookupInRemappedAssembly = ssToLookupInRemappedAssembly.stream().map(AccessionedDocument::getAccession) - .collect(Collectors.toList()); + .collect(Collectors.toList()); Map> ssInRemappedAssemblyGroupedByID = submittedVariantAccessioningService .getAllActiveByAssemblyAndAccessionIn(remappedAssembly, ssIDsToLookupInRemappedAssembly) .stream() .map(entity -> new SubmittedVariantEntity(entity.getAccession(), entity.getHash(), - entity.getData(), - entity.getVersion())) + entity.getData(), + entity.getVersion())) .filter(entity -> Objects.nonNull(entity.getClusteredVariantAccession())) .collect(Collectors.groupingBy(SubmittedVariantEntity::getAccession)); backpropagateRSToSS(ssToLookupInRemappedAssembly, ssInRemappedAssemblyGroupedByID); @@ -109,21 +107,21 @@ private void backpropagateRSToSS( // Updates to Submitted Variant Entity (SVE) collection with the RS created above BulkOperations evaSVEUpdates = mongoTemplate.bulkOps(BulkOperations.BulkMode.UNORDERED, - SubmittedVariantEntity.class); + SubmittedVariantEntity.class); BulkOperations dbsnpSVEUpdates = mongoTemplate.bulkOps(BulkOperations.BulkMode.UNORDERED, - DbsnpSubmittedVariantEntity.class); + DbsnpSubmittedVariantEntity.class); // Inserts to Submitted Variant Operation Entity (SVOE) collection recording the RS updates made above BulkOperations evaSVOEInserts = mongoTemplate.bulkOps(BulkOperations.BulkMode.UNORDERED, - SubmittedVariantOperationEntity.class); + SubmittedVariantOperationEntity.class); BulkOperations dbsnpSVOEInserts = mongoTemplate.bulkOps(BulkOperations.BulkMode.UNORDERED, - DbsnpSubmittedVariantOperationEntity.class); + DbsnpSubmittedVariantOperationEntity.class); int numEvaRSAssignments = 0; int numDbsnpRSAssignments = 0; - for (SubmittedVariantEntity submittedVariantEntity: submittedVariantEntitiesInOriginalAssemblyWithNoRS) { + for (SubmittedVariantEntity submittedVariantEntity : submittedVariantEntitiesInOriginalAssemblyWithNoRS) { Long ssIDToBeClustered = submittedVariantEntity.getAccession(); Long rsInOriginalAssembly = submittedVariantEntity.getClusteredVariantAccession(); Long rsToBackPropagate = null; diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/io/ClusteringWriter.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/io/ClusteringWriter.java index 6bc7b8617..adc0ba730 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/io/ClusteringWriter.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/io/ClusteringWriter.java @@ -17,13 +17,14 @@ import com.mongodb.MongoBulkWriteException; import htsjdk.samtools.util.StringUtil; +import jakarta.annotation.Nonnull; import org.springframework.batch.core.JobExecution; +import org.springframework.batch.item.Chunk; import org.springframework.batch.item.ItemWriter; import org.springframework.data.mongodb.core.BulkOperations; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.data.mongodb.core.query.Query; import org.springframework.data.mongodb.core.query.Update; -import org.springframework.data.util.CloseableIterator; import org.springframework.util.Assert; import uk.ac.ebi.ampt2d.commons.accession.core.exceptions.AccessionCouldNotBeGeneratedException; import uk.ac.ebi.ampt2d.commons.accession.core.exceptions.AccessionDoesNotExistException; @@ -32,7 +33,6 @@ import uk.ac.ebi.ampt2d.commons.accession.core.models.GetOrCreateAccessionWrapper; import uk.ac.ebi.ampt2d.commons.accession.hashing.SHA1HashingFunction; import uk.ac.ebi.ampt2d.commons.accession.persistence.mongodb.document.EventDocument; - import uk.ac.ebi.eva.accession.clustering.metric.ClusteringMetric; import uk.ac.ebi.eva.accession.core.EVAObjectModelUtils; import uk.ac.ebi.eva.accession.core.model.ClusteredVariant; @@ -53,7 +53,6 @@ import uk.ac.ebi.eva.commons.core.models.VariantClassifier; import uk.ac.ebi.eva.metrics.metric.MetricCompute; -import javax.annotation.Nonnull; import java.io.File; import java.io.FileWriter; import java.io.IOException; @@ -68,16 +67,18 @@ import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; +import java.util.stream.Stream; import static org.springframework.data.mongodb.core.query.Criteria.where; import static org.springframework.data.mongodb.core.query.Query.query; -import static uk.ac.ebi.eva.accession.clustering.configuration.batch.io.RSMergeAndSplitCandidatesReaderConfiguration.*; +import static uk.ac.ebi.eva.accession.clustering.configuration.batch.io.RSMergeAndSplitCandidatesReaderConfiguration.MERGE_CANDIDATE_ID_PREFIX; +import static uk.ac.ebi.eva.accession.clustering.configuration.batch.io.RSMergeAndSplitCandidatesReaderConfiguration.SPLIT_CANDIDATE_ID_PREFIX; /** * This writer has two parts: * 1. Use the accessioning service to generate new RS IDs or get existing ones * 2. Update the submitted variants to include the "rs" field with the generated/retrieved accessions - * + *

* Some edge cases take into account if a clustered variant is multimap. The definition of multimap variants that this * class uses is "clustered variants whose mapWeight is 2 or greater". Another definition is "clustered variants * whose accession maps several times in the same assembly". Although both definitions should yield the same @@ -141,16 +142,15 @@ public ClusteringWriter(MongoTemplate mongoTemplate, private void getSVOEWithMergeAndRSSplitCandidates() { List MERGE_AND_SPLIT_EVENTS = Arrays.asList(EventType.RS_MERGE_CANDIDATES.name(), - EventType.RS_SPLIT_CANDIDATES.name()); + EventType.RS_SPLIT_CANDIDATES.name()); Query queryOperations = query(where("eventType").in(MERGE_AND_SPLIT_EVENTS) - .and("inactiveObjects").elemMatch(where("seq").is(assembly))); + .and("inactiveObjects").elemMatch(where("seq").is(assembly))); mergeCandidateSVOE = new HashMap<>(); rsSplitCandidateSVOE = new HashMap<>(); - try ( - CloseableIterator svoes = mongoTemplate.stream( - queryOperations, SubmittedVariantOperationEntity.class)) { - svoes.forEachRemaining(svoe -> { + try (Stream svoes = mongoTemplate.stream( + queryOperations, SubmittedVariantOperationEntity.class)) { + svoes.forEach(svoe -> { if (svoe.getEventType().equals(EventType.RS_MERGE_CANDIDATES)) { mergeCandidateSVOE.put(getClusteredVariantHash(svoe.getInactiveObjects().get(0).getModel()), svoe); } else if (svoe.getEventType().equals(EventType.RS_SPLIT_CANDIDATES)) { @@ -161,7 +161,7 @@ private void getSVOEWithMergeAndRSSplitCandidates() { } @Override - public void write(@Nonnull List submittedVariantEntities) + public void write(@Nonnull Chunk submittedVariantEntities) throws IOException, MongoBulkWriteException, AccessionCouldNotBeGeneratedException, AccessionDoesNotExistException { try { @@ -175,27 +175,26 @@ public void write(@Nonnull List submittedVaria if (!this.processClusteredRemappedVariants) { clusterSubmittedVariants(submittedVariantEntities); } - } - finally { + } finally { this.rsReportFileWriter.close(); } } - private void getOrCreateClusteredVariantAccessions(List submittedVariantEntities) + private void getOrCreateClusteredVariantAccessions(Chunk submittedVariantEntities) throws AccessionCouldNotBeGeneratedException, IOException { if (processClusteredRemappedVariants) { processClusteredRemappedVariants(submittedVariantEntities); } else { - List clusteredVariants = submittedVariantEntities.stream() - .map(this::toClusteredVariant) - .collect(Collectors.toList()); + List clusteredVariants = submittedVariantEntities.getItems().stream() + .map(this::toClusteredVariant) + .collect(Collectors.toList()); if (!clusteredVariants.isEmpty()) { List> accessionWrappers = clusteredService.getOrCreate(clusteredVariants, jobExecution.getJobId().toString()); for (GetOrCreateAccessionWrapper result : accessionWrappers) { if (result.isNewAccession()) { ClusteringWriter.writeRSReportEntry(this.rsReportFileWriter, result.getAccession(), - result.getHash()); + result.getHash()); } } List> accessionNoMultimap = @@ -216,28 +215,28 @@ private void getOrCreateClusteredVariantAccessions(List * --------------------Before Clustering-------------------- - * - * SubmittedVariantEntity + *

+ * SubmittedVariantEntity * SS RS ASM STUDY CONTIG POS REF ALT * 500 306 ASM1 PRJEB1 Chr1 1000 A T (original) * 501 306 ASM1 PRJEB2 Chr1 1000 A T (original) * 500 306 ASM2 PRJEB1 Chr1 1000 A T (remapped) * 501 306 ASM2 PRJEB2 Chr1 1500 A T (remapped) - * - * ClusteredVariantEntity + *

+ * ClusteredVariantEntity * RS HASH ASM POS CONTIG TYPE * 306 ASM1_Chr1_1000_SNV ASM1 1000 Chr1 SNV - * + *

* SS id 500 and 501 has same RS because of remapping, but they are now at different positions and can't have same RS id. * These needs to be identified and stored in submittedVariantOperationEntity table for rectification. - * + *

* --------------------After Detection-------------------- - * - * SubmittedVariantOperationEntity + *

+ * SubmittedVariantOperationEntity * ACCESSION EVENT_TYPE REASON INACTIVe_OBJECTS * 306 RS_SPLIT_CANDIDATE Hash mismatch with 306 {ss-500 and ss-501} */ - private void processClusteredRemappedVariants(List submittedVariants) + private void processClusteredRemappedVariants(Chunk submittedVariants) throws IOException { List clusteredRemappedSubmittedVariants = getClusteredAndRemappedVariants(submittedVariants); if (clusteredRemappedSubmittedVariants.isEmpty()) { @@ -249,8 +248,8 @@ private void processClusteredRemappedVariants(List updateRsSplitCandidateSVOE = new HashMap<>(); Set rsIDs = clusteredRemappedSubmittedVariants.stream() - .map(SubmittedVariantEntity::getClusteredVariantAccession) - .collect(Collectors.toSet()); + .map(SubmittedVariantEntity::getClusteredVariantAccession) + .collect(Collectors.toSet()); // Initialize a map of "RS -> list of associated hashes" with hashes for RS IDs already present in the database // We will keep updating this map when we encounter new RS IDs present in the remapped variants SS record // but with no corresponding RS record in the database @@ -262,12 +261,12 @@ private void processClusteredRemappedVariants(List()); checkIfCandidateForRSSplit(remappedSubmittedVariantEntity, clusteredVariantEntity, - allExistingHashesGroupByRS, updateRsSplitCandidateSVOE); + allExistingHashesGroupByRS, updateRsSplitCandidateSVOE); if (!isExistingRSOrMergeCandidate) { if (clusteredVariantEntity.getAccession() >= accessioningMonotonicInitRs) { @@ -276,17 +275,17 @@ private void processClusteredRemappedVariants(List getClusteredAndRemappedVariants(List submittedVariants) { - return submittedVariants.stream() + private List getClusteredAndRemappedVariants(Chunk submittedVariants) { + return submittedVariants.getItems().stream() .filter(sve -> Objects.nonNull(sve.getClusteredVariantAccession())) .filter(sve -> !StringUtil.isBlank(sve.getRemappedFrom())) .filter(sve -> Objects.isNull(sve.getMapWeight())) @@ -298,7 +297,7 @@ private Map getSubmittedVariantsAllExistingHashesInDB(List> getAllHashesForAssemblyAndRSInDB(List accessionList) { @@ -353,9 +352,9 @@ private boolean checkIfExistingRSOrCandidateForMerge(SubmittedVariantEntity subm } else { List inactiveObjects = getAllSubmittedVariantsWithClusteringAccession(accessionInDB).stream() - .filter(sve-> toClusteredVariantEntity(sve).getHashedMessage().equals(variantHash)) - .map(SubmittedVariantInactiveEntity::new) - .collect(Collectors.toList()); + .filter(sve -> toClusteredVariantEntity(sve).getHashedMessage().equals(variantHash)) + .map(SubmittedVariantInactiveEntity::new) + .collect(Collectors.toList()); inactiveObjects.add(new SubmittedVariantInactiveEntity(submittedVariantEntity)); submittedVariantOperationEntity = new SubmittedVariantOperationEntity(); submittedVariantOperationEntity.fill(EventType.RS_MERGE_CANDIDATES, accessionInDB, @@ -395,8 +394,8 @@ private void checkIfCandidateForRSSplit(SubmittedVariantEntity submittedVariantE submittedVariantOperationEntity = new SubmittedVariantOperationEntity(); List inactiveEntities = getAllSubmittedVariantsWithClusteringAccession(variantAccession).stream() - .map(SubmittedVariantInactiveEntity::new) - .collect(Collectors.toList()); + .map(SubmittedVariantInactiveEntity::new) + .collect(Collectors.toList()); submittedVariantOperationEntity.fill(EventType.RS_SPLIT_CANDIDATES, variantAccession, "Hash mismatch with " + variantAccession, inactiveEntities); rsSplitCandidateSVOE.put(variantAccession, submittedVariantOperationEntity); @@ -433,7 +432,7 @@ private void insertAllEntriesInDB(List clusteredVariantE for (Map.Entry entry : mergeSVOE.entrySet()) { SubmittedVariantOperationEntity svoe = entry.getValue(); - if(Objects.isNull(svoe.getId())){ + if (Objects.isNull(svoe.getId())) { svoe.setId(getMergeCandidateId(svoe)); mergeSVOEInsertEntries.add(svoe); continue; @@ -451,7 +450,7 @@ private void insertAllEntriesInDB(List clusteredVariantE for (Map.Entry entry : rsSplitSVOE.entrySet()) { Long accession = entry.getKey(); SubmittedVariantOperationEntity svoe = entry.getValue(); - if(Objects.isNull(svoe.getId())){ + if (Objects.isNull(svoe.getId())) { svoe.setId(getSplitCandidateId(svoe)); rsSplitSVOEInsertEntries.add(svoe); continue; @@ -478,13 +477,13 @@ public ClusteredVariantEntity toClusteredVariantEntity(SubmittedVariantEntity su private ClusteredVariant toClusteredVariant(ISubmittedVariant submittedVariant) { return new ClusteredVariant(submittedVariant.getReferenceSequenceAccession(), - submittedVariant.getTaxonomyAccession(), - submittedVariant.getContig(), - submittedVariant.getStart(), - VariantClassifier.getVariantClassification(submittedVariant.getReferenceAllele(), - submittedVariant.getAlternateAllele()), - submittedVariant.isValidated(), - submittedVariant.getCreatedDate()); + submittedVariant.getTaxonomyAccession(), + submittedVariant.getContig(), + submittedVariant.getStart(), + VariantClassifier.getVariantClassification(submittedVariant.getReferenceAllele(), + submittedVariant.getAlternateAllele()), + submittedVariant.isValidated(), + submittedVariant.getCreatedDate()); } public Class> @@ -505,7 +504,7 @@ private boolean isEvaClusteredAccession(Long accession) { * In EVA-2003 we decided not to merge any RS ID that maps to several places in the same assembly * (mapping weight > 1) as this might be a signal of a low quality variant, and merging other "real" variants * in the same RS would just make things more complicated. - * + *

* Note that for submitted variants the test is not this simple, as 1:1000:A:T and 1:1000:A:G can be present in the * same assembly and still not classify as multimap. */ @@ -519,7 +518,7 @@ protected boolean isMultimap(IClusteredVariant clusteredVariant) { /** * From EVA-2071, do not cluster submitted variants into a multimap clustered variant. - * + *

* This function removes candidate clustered variant accessions if they are multimap. This means that some submitted * variants will be kept unclustered. This potentially will be revisited in the future, but for now (release 2) we * are leaving this out of scope. @@ -532,11 +531,11 @@ private List> exclu /** * This function assigns a clustered variant accession (rs) to the submitted variants that didn't have any. */ - private void clusterSubmittedVariants(List submittedVariantEntities) { + private void clusterSubmittedVariants(Chunk submittedVariantEntities) { BulkOperations bulkOperations = mongoTemplate.bulkOps(BulkOperations.BulkMode.UNORDERED, - SubmittedVariantEntity.class); + SubmittedVariantEntity.class); BulkOperations dbsnpBulkOperations = mongoTemplate.bulkOps(BulkOperations.BulkMode.UNORDERED, - DbsnpSubmittedVariantEntity.class); + DbsnpSubmittedVariantEntity.class); BulkOperations bulkHistoryOperations = mongoTemplate.bulkOps(BulkOperations.BulkMode.UNORDERED, SubmittedVariantOperationEntity.class); BulkOperations dbsnpBulkHistoryOperations = mongoTemplate.bulkOps(BulkOperations.BulkMode.UNORDERED, @@ -609,21 +608,21 @@ protected String getClusteredVariantHash(ISubmittedVariant submittedVariant) { return clusteredHashingFunction.apply(clusteredVariant); } - protected static void writeRSReportEntry (FileWriter rsReportWriter, Long rsAccession, String rsHash) + protected static void writeRSReportEntry(FileWriter rsReportWriter, Long rsAccession, String rsHash) throws IOException { rsReportWriter.write(String.format("%s\t%s\n", rsAccession, rsHash)); } public static String getMergeCandidateId(SubmittedVariantOperationEntity svoe) { return String.format("%s_%s_%s", MERGE_CANDIDATE_ID_PREFIX, - svoe.getInactiveObjects().get(0).getReferenceSequenceAccession(), - EVAObjectModelUtils.getClusteredVariantHash(svoe.getInactiveObjects().get(0))); + svoe.getInactiveObjects().get(0).getReferenceSequenceAccession(), + EVAObjectModelUtils.getClusteredVariantHash(svoe.getInactiveObjects().get(0))); } public static String getSplitCandidateId(SubmittedVariantOperationEntity svoe) { return String.format("%s_%s_%d", SPLIT_CANDIDATE_ID_PREFIX, - svoe.getInactiveObjects().get(0).getReferenceSequenceAccession(), - svoe.getInactiveObjects().get(0).getClusteredVariantAccession()); + svoe.getInactiveObjects().get(0).getReferenceSequenceAccession(), + svoe.getInactiveObjects().get(0).getClusteredVariantAccession()); } public void setJobExecution(JobExecution jobExecution) { diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/io/ListOfListItemWriter.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/io/ListOfListItemWriter.java index f4c270667..ee8479bb2 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/io/ListOfListItemWriter.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/io/ListOfListItemWriter.java @@ -15,6 +15,7 @@ */ package uk.ac.ebi.eva.accession.clustering.batch.io; +import org.springframework.batch.item.Chunk; import org.springframework.batch.item.ItemStreamWriter; import org.springframework.batch.item.ItemWriter; @@ -32,14 +33,14 @@ public ListOfListItemWriter(ItemWriter itemWriter) { } @Override - public void write(List> listOfLists) throws Exception { + public void write(Chunk> listOfLists) throws Exception { if (listOfLists.isEmpty()) { return; } - List all = listOfLists.stream().flatMap(Collection::stream).collect(Collectors.toList()); + List all = listOfLists.getItems().stream().flatMap(Collection::stream).collect(Collectors.toList()); - itemWriter.write(all); + itemWriter.write(new Chunk<>(all)); } public void setItemWriter(ItemStreamWriter itemWriter) { diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/io/RSMergeWriter.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/io/RSMergeWriter.java index 7996c8e2b..af6a780b6 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/io/RSMergeWriter.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/io/RSMergeWriter.java @@ -16,9 +16,11 @@ package uk.ac.ebi.eva.accession.clustering.batch.io; import com.mongodb.MongoBulkWriteException; +import jakarta.annotation.Nonnull; import org.apache.commons.lang3.tuple.ImmutablePair; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.batch.item.Chunk; import org.springframework.batch.item.ItemWriter; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.data.mongodb.core.query.Criteria; @@ -45,7 +47,6 @@ import uk.ac.ebi.eva.commons.core.models.contigalias.ContigNamingConvention; import uk.ac.ebi.eva.metrics.metric.MetricCompute; -import javax.annotation.Nonnull; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -126,9 +127,9 @@ public RSMergeWriter(ClusteringWriter clusteringWriter, MongoTemplate mongoTempl } @Override - public void write(@Nonnull List submittedVariantOperationEntities) + public void write(@Nonnull Chunk submittedVariantOperationEntities) throws MongoBulkWriteException, AccessionCouldNotBeGeneratedException, AccessionDoesNotExistException { - allMergeCandidateOperations = new ArrayList<>(submittedVariantOperationEntities); + allMergeCandidateOperations = new ArrayList<>(submittedVariantOperationEntities.getItems()); // Create a map of all merge candidate operations keyed by RS ID to facilitate quick random lookups populateOperationsIndex(allMergeCandidateOperations); for (int i = 0; i < allMergeCandidateOperations.size(); i++) { @@ -136,9 +137,9 @@ public void write(@Nonnull List submi writeRSMerge(allMergeCandidateOperations.get(i)); } List allCandidateIds = allMergeCandidateOperations.stream().map(EventDocument::getId) - .collect(Collectors.toList()); + .collect(Collectors.toList()); this.mongoTemplate.findAllAndRemove(query(where("_id").in(allCandidateIds)), - SubmittedVariantOperationEntity.class); + SubmittedVariantOperationEntity.class); } private void populateOperationsIndex @@ -146,10 +147,10 @@ public void write(@Nonnull List submi rsIDIndexedOperations = new HashMap<>(); for (int i = 0; i < allMergeCandidateOperations.size(); i++) { SubmittedVariantOperationEntity operation = allMergeCandidateOperations.get(i); - for (Long rsID: + for (Long rsID : operation.getInactiveObjects().stream() - .map(SubmittedVariantInactiveEntity::getClusteredVariantAccession) - .collect(Collectors.toSet())) { + .map(SubmittedVariantInactiveEntity::getClusteredVariantAccession) + .collect(Collectors.toSet())) { OperationWithIndex obj = new OperationWithIndex(i, operation); if (rsIDIndexedOperations.containsKey(rsID)) { rsIDIndexedOperations.get(rsID).add(obj); @@ -176,13 +177,13 @@ public void writeRSMerge(SubmittedVariantOperationEntity currentOperation) // because ClusteredVariantEntity "equals" method does NOT involve comparing accessions List mergeCandidates = currentOperation.getInactiveObjects() - .stream() - // Ensure duplicates inside inactiveObjects are tolerated - .filter(distinctByKey(this::getHashedMessageAndAccessionForSVIE)) - .map(entity -> clusteringWriter.toClusteredVariantEntity( - entity.toSubmittedVariantEntity())) - .filter(distinctByKey(AccessionedDocument::getAccession)) - .collect(Collectors.toList()); + .stream() + // Ensure duplicates inside inactiveObjects are tolerated + .filter(distinctByKey(this::getHashedMessageAndAccessionForSVIE)) + .map(entity -> clusteringWriter.toClusteredVariantEntity( + entity.toSubmittedVariantEntity())) + .filter(distinctByKey(AccessionedDocument::getAccession)) + .collect(Collectors.toList()); // From among the participating RS in a merge, // use the current RS prioritization policy to get the target RS into which the rest of the RS will be merged ImmutablePair> mergeDestinationAndMergees = @@ -192,9 +193,9 @@ public void writeRSMerge(SubmittedVariantOperationEntity currentOperation) removeMergeesAndInsertMergeDestination(mergeDestination, mergees); - for (ClusteredVariantEntity mergee: mergees) { + for (ClusteredVariantEntity mergee : mergees) { logger.info("RS merge operation: Merging rs{} to rs{} due to hash collision...", - mergee.getAccession(), mergeDestination.getAccession()); + mergee.getAccession(), mergeDestination.getAccession()); recordMergeOperations(mergeDestination, mergee, currentOperation); } } @@ -234,17 +235,17 @@ private void insertMergeOperation(ClusteredVariantEntity mergeDestination, Clust List> existingOperations = this.mongoTemplate.find(query(where(ACCESSION_ATTRIBUTE).is(mergee.getAccession())) - .addCriteria(where(MERGE_DESTINATION_ATTRIBUTE).is( - mergeDestination.getAccession())) - .addCriteria(where(ASM_ATTRIBUTE_IN_OPERATIONS_COLLECTION) - .is(this.assemblyAccession)), - operationsCollectionToWriteTo); + .addCriteria(where(MERGE_DESTINATION_ATTRIBUTE).is( + mergeDestination.getAccession())) + .addCriteria(where(ASM_ATTRIBUTE_IN_OPERATIONS_COLLECTION) + .is(this.assemblyAccession)), + operationsCollectionToWriteTo); if (existingOperations.isEmpty()) { ClusteredVariantOperationEntity operation = new ClusteredVariantOperationEntity(); operation.fill(EventType.MERGED, mergee.getAccession(), mergeDestination.getAccession(), - "After remapping to " + mergee.getAssemblyAccession() + - ", RS IDs mapped to the same locus.", - Collections.singletonList(new ClusteredVariantInactiveEntity(mergee))); + "After remapping to " + mergee.getAssemblyAccession() + + ", RS IDs mapped to the same locus.", + Collections.singletonList(new ClusteredVariantInactiveEntity(mergee))); this.mongoTemplate.insert(operation, this.mongoTemplate.getCollectionName(operationsCollectionToWriteTo)); metricCompute.addCount(ClusteringMetric.CLUSTERED_VARIANTS_MERGE_OPERATIONS, 1); @@ -261,6 +262,7 @@ private void insertRSRecordForMergeDestination(ClusteredVariantEntity mergeDesti /** * Get destination RS and the set of RS to be merged into the destination RS + * * @param mergeCandidates Set of RS candidates that should be merged * @return Pair with the first element being the destination RS * and the next being the list of RS that should be merged into the former. @@ -274,17 +276,17 @@ private ImmutablePair> getM } final Long targetRSAccession = lastPrioritizedAccession; ClusteredVariantEntity targetRS = mergeCandidates.stream().filter(rs -> rs.getAccession() - .equals(targetRSAccession)) - .findFirst().get(); + .equals(targetRSAccession)) + .findFirst().get(); List mergees = mergeCandidates.stream() - .filter(rs -> !rs.getAccession() - .equals(targetRSAccession)) - .collect(Collectors.toList()); + .filter(rs -> !rs.getAccession() + .equals(targetRSAccession)) + .collect(Collectors.toList()); return new ImmutablePair<>(targetRS, mergees); } protected void recordMergeOperations(ClusteredVariantEntity mergeDestination, ClusteredVariantEntity mergee, - SubmittedVariantOperationEntity currentOperation) { + SubmittedVariantOperationEntity currentOperation) { Long accessionToBeMerged = mergee.getAccession(); Long accessionToKeep = mergeDestination.getAccession(); @@ -296,9 +298,9 @@ protected void recordMergeOperations(ClusteredVariantEntity mergeDestination, Cl ClusteredVariantMergingPolicy.Priority prioritised = new ClusteredVariantMergingPolicy.Priority(accessionToKeep, accessionToBeMerged); updateSubmittedVariants(prioritised, currentOperation, SubmittedVariantEntity.class, - SubmittedVariantOperationEntity.class); + SubmittedVariantOperationEntity.class); updateSubmittedVariants(prioritised, currentOperation, DbsnpSubmittedVariantEntity.class, - DbsnpSubmittedVariantOperationEntity.class); + DbsnpSubmittedVariantOperationEntity.class); // Update other merge candidate operations involving the mergee // by replacing references to mergee RS ID with the target RS ID @@ -325,36 +327,36 @@ private void updateOperationsInDB(ClusteredVariantMergingPolicy.Priority priorit where(RS_KEY_IN_OPERATIONS_COLLECTION).is(prioritised.accessionToBeMerged))); List operationsInDBInvolvingMergee = mongoTemplate.find(queryForMergeCandidatesInvolvingMergee, SubmittedVariantOperationEntity.class); - for (SubmittedVariantOperationEntity operation: operationsInDBInvolvingMergee) { + for (SubmittedVariantOperationEntity operation : operationsInDBInvolvingMergee) { List submittedVariantInactiveEntitiesWithMergeeRSReplaced = operation.getInactiveObjects().stream() - .map(entity -> replaceRSInSubmittedVariantInactiveEntity( - entity, prioritised.accessionToBeMerged, prioritised.accessionToKeep) - ).collect(Collectors.toList()); + .map(entity -> replaceRSInSubmittedVariantInactiveEntity( + entity, prioritised.accessionToBeMerged, prioritised.accessionToKeep) + ).collect(Collectors.toList()); mongoTemplate.updateFirst(query(where(ID_ATTRIBUTE).is(operation.getId())), - update(INACTIVE_OBJECT_ATTRIBUTE, - submittedVariantInactiveEntitiesWithMergeeRSReplaced), - SubmittedVariantOperationEntity.class); + update(INACTIVE_OBJECT_ATTRIBUTE, + submittedVariantInactiveEntitiesWithMergeeRSReplaced), + SubmittedVariantOperationEntity.class); } } private void updateOperationsInMemory(ClusteredVariantMergingPolicy.Priority prioritised, - SubmittedVariantOperationEntity currentMergeOperation) { + SubmittedVariantOperationEntity currentMergeOperation) { if (rsIDIndexedOperations.containsKey(prioritised.accessionToBeMerged)) { List operationsInMemoryInvolvingMergee = rsIDIndexedOperations.get(prioritised.accessionToBeMerged) - .stream() - // Filter for upcoming operations only - .filter(e -> (e.operationIndex > this.currentlyProcessingOperationIndex) - && !(e.operation.equals(currentMergeOperation))) - .collect(Collectors.toList()); - for (OperationWithIndex operationWithIndex: operationsInMemoryInvolvingMergee) { + .stream() + // Filter for upcoming operations only + .filter(e -> (e.operationIndex > this.currentlyProcessingOperationIndex) + && !(e.operation.equals(currentMergeOperation))) + .collect(Collectors.toList()); + for (OperationWithIndex operationWithIndex : operationsInMemoryInvolvingMergee) { SubmittedVariantOperationEntity operation = operationWithIndex.operation; List inactiveEntities = operation.getInactiveObjects().stream() - .map(entity -> replaceRSInSubmittedVariantInactiveEntity - (entity, prioritised.accessionToBeMerged, prioritised.accessionToKeep)) - .collect(Collectors.toList()); + .map(entity -> replaceRSInSubmittedVariantInactiveEntity + (entity, prioritised.accessionToBeMerged, prioritised.accessionToKeep)) + .collect(Collectors.toList()); SubmittedVariantOperationEntity updatedOperation = new SubmittedVariantOperationEntity(); updatedOperation.fill(operation.getEventType(), operation.getAccession(), operation.getReason(), @@ -377,11 +379,11 @@ private SubmittedVariantInactiveEntity replaceRSInSubmittedVariantInactiveEntity private void updateSplitCandidates(ClusteredVariantMergingPolicy.Priority prioritised) { Query queryForSplitCandidatesInvolvingTargetRS = query(getSplitCandidatesCriteria(this.assemblyAccession) - .and(RS_KEY_IN_OPERATIONS_COLLECTION) - .is(prioritised.accessionToKeep)); + .and(RS_KEY_IN_OPERATIONS_COLLECTION) + .is(prioritised.accessionToKeep)); Query queryForSplitCandidatesInvolvingMergee = query(getSplitCandidatesCriteria(this.assemblyAccession) - .and(RS_KEY_IN_OPERATIONS_COLLECTION) - .is(prioritised.accessionToBeMerged)); + .and(RS_KEY_IN_OPERATIONS_COLLECTION) + .is(prioritised.accessionToBeMerged)); // Since the mergee has been merged into the target RS, // the split candidates record for mergee are no longer valid - so, delete them! mongoTemplate.remove(queryForSplitCandidatesInvolvingMergee, SubmittedVariantOperationEntity.class); @@ -393,30 +395,30 @@ private void updateSplitCandidates(ClusteredVariantMergingPolicy.Priority priori List ssClusteredUnderTargetRS = this.submittedVariantAccessioningService .getByClusteredVariantAccessionIn(Collections.singletonList(prioritised.accessionToKeep), - ContigNamingConvention.NO_REPLACEMENT) + ContigNamingConvention.NO_REPLACEMENT) .stream() .filter(result -> result.getData().getReferenceSequenceAccession() - .equals(this.assemblyAccession)) + .equals(this.assemblyAccession)) .map(result -> new SubmittedVariantEntity(result.getAccession(), result.getHash(), - result.getData(), result.getVersion())) + result.getData(), result.getVersion())) .map(SubmittedVariantInactiveEntity::new) .collect(Collectors.toList()); Map> distinctLociInDBForTargetRS = ssClusteredUnderTargetRS.stream() - .map(SubmittedVariantInactiveEntity::toSubmittedVariantEntity) - .map(clusteringWriter::toClusteredVariantEntity) - .collect(Collectors.groupingBy(ClusteredVariantEntity::getHashedMessage)); + .map(SubmittedVariantInactiveEntity::toSubmittedVariantEntity) + .map(clusteringWriter::toClusteredVariantEntity) + .collect(Collectors.groupingBy(ClusteredVariantEntity::getHashedMessage)); //Update existing split candidates record for target RS if it exists. Else, create a new record! if (Objects.nonNull(splitCandidateInvolvingTargetRS) && distinctLociInDBForTargetRS.size() > 1) { mongoTemplate.updateFirst(query(where(ID_ATTRIBUTE).is(splitCandidateInvolvingTargetRS.getId())), - update(INACTIVE_OBJECT_ATTRIBUTE, ssClusteredUnderTargetRS), - SubmittedVariantOperationEntity.class); + update(INACTIVE_OBJECT_ATTRIBUTE, ssClusteredUnderTargetRS), + SubmittedVariantOperationEntity.class); } else { Set targetRSDistinctLoci = ssClusteredUnderTargetRS.stream().map(entity -> clusteringWriter - .getClusteredVariantHash(entity.getModel())) - .collect(Collectors.toSet()); + .getClusteredVariantHash(entity.getModel())) + .collect(Collectors.toSet()); // Condition for generating split operation: ensure that there is more than one locus sharing the target RS if (targetRSDistinctLoci.size() > 1) { // The new SPLIT candidate is for rs prioritised.accessionToKeep @@ -458,8 +460,8 @@ private void updateSubmittedVariants( List operations = svToUpdate.stream() - .map(sv -> buildSubmittedOperation(sv, prioritised.accessionToKeep)) - .collect(Collectors.toList()); + .map(sv -> buildSubmittedOperation(sv, prioritised.accessionToKeep)) + .collect(Collectors.toList()); mongoTemplate.insert(operations, submittedOperationCollection); metricCompute.addCount(ClusteringMetric.SUBMITTED_VARIANTS_UPDATE_OPERATIONS, operations.size()); } diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/io/RSSplitWriter.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/io/RSSplitWriter.java index 41b89f64e..21dbe5645 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/io/RSSplitWriter.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/io/RSSplitWriter.java @@ -16,22 +16,21 @@ package uk.ac.ebi.eva.accession.clustering.batch.io; import com.mongodb.MongoBulkWriteException; - import com.mongodb.client.result.UpdateResult; +import jakarta.annotation.Nonnull; import org.apache.commons.lang3.tuple.ImmutablePair; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.batch.core.JobExecution; +import org.springframework.batch.item.Chunk; import org.springframework.batch.item.ItemWriter; import org.springframework.data.mongodb.core.MongoTemplate; - import org.springframework.data.mongodb.core.query.Query; import org.springframework.data.mongodb.core.query.Update; import uk.ac.ebi.ampt2d.commons.accession.core.exceptions.AccessionCouldNotBeGeneratedException; import uk.ac.ebi.ampt2d.commons.accession.core.models.AccessionWrapper; import uk.ac.ebi.ampt2d.commons.accession.core.models.EventType; import uk.ac.ebi.ampt2d.commons.accession.persistence.mongodb.document.EventDocument; - import uk.ac.ebi.eva.accession.clustering.batch.io.ClusteredVariantSplittingPolicy.SplitDeterminants; import uk.ac.ebi.eva.accession.clustering.metric.ClusteringMetric; import uk.ac.ebi.eva.accession.core.model.IClusteredVariant; @@ -50,7 +49,6 @@ import uk.ac.ebi.eva.commons.core.models.contigalias.ContigNamingConvention; import uk.ac.ebi.eva.metrics.metric.MetricCompute; -import javax.annotation.Nonnull; import java.io.File; import java.io.FileWriter; import java.io.IOException; @@ -114,15 +112,14 @@ public RSSplitWriter(ClusteringWriter clusteringWriter, } @Override - public void write(@Nonnull List submittedVariantOperationEntities) + public void write(@Nonnull Chunk submittedVariantOperationEntities) throws MongoBulkWriteException, AccessionCouldNotBeGeneratedException, IOException { try { this.rsReportFileWriter = new FileWriter(this.rsReportFile, true); for (SubmittedVariantOperationEntity entity : submittedVariantOperationEntities) { writeRSSplit(entity); } - } - finally { + } finally { this.rsReportFileWriter.close(); } } @@ -159,17 +156,17 @@ public void writeRSSplit(SubmittedVariantOperationEntity submittedVariantOperati .entrySet().stream().map(rsHashAndAssociatedSS -> new SplitDeterminants( clusteringWriter.toClusteredVariantEntity(rsHashAndAssociatedSS.getValue() - .get(0)), + .get(0)), rsHashAndAssociatedSS.getKey(), rsHashAndAssociatedSS.getValue().size(), // Get lowest SS ID associated with a given RS hash rsHashAndAssociatedSS.getValue().stream() - .map(SubmittedVariantEntity::getAccession) - .min(Comparator.naturalOrder()).get())) + .map(SubmittedVariantEntity::getAccession) + .min(Comparator.naturalOrder()).get())) .collect(Collectors.toList()); // Based on the split policy, one of the hashes will retain the RS associated with it // and the other hashes should be associated with new RS IDs - List hashesThatShouldGetNewRS = getHashesThatShouldGetNewRS(splitCandidates); + List hashesThatShouldGetNewRS = getHashesThatShouldGetNewRS(splitCandidates); issueNewRSForHashes(hashesThatShouldGetNewRS, submittedVariantOperationEntity.getInactiveObjects()); } @@ -182,13 +179,13 @@ private void issueNewRSForHashes(List hashesThatShouldGetNewRS, throws AccessionCouldNotBeGeneratedException, IOException { Map> rsHashAndAssociatedSS = submittedVariantInactiveEntities - .stream() - .map(SubmittedVariantInactiveEntity::toSubmittedVariantEntity) - // If the split was processed multiple times due to multiple runs of the clustering pipeline - // ensure that the SS was not already assigned a new RS - .filter(ss -> !this.doesSSAlreadyHaveANewRS(ss)) - .collect(Collectors.groupingBy(this::getRSHashForSS)); - for (String rsHash: rsHashAndAssociatedSS.keySet()) { + .stream() + .map(SubmittedVariantInactiveEntity::toSubmittedVariantEntity) + // If the split was processed multiple times due to multiple runs of the clustering pipeline + // ensure that the SS was not already assigned a new RS + .filter(ss -> !this.doesSSAlreadyHaveANewRS(ss)) + .collect(Collectors.groupingBy(this::getRSHashForSS)); + for (String rsHash : rsHashAndAssociatedSS.keySet()) { if (hashesThatShouldGetNewRS.contains(rsHash)) { // Remove entry in clustered variant collections if hash already exists removeExistingHash(rsHash); @@ -197,19 +194,19 @@ private void issueNewRSForHashes(List hashesThatShouldGetNewRS, rsHashAndAssociatedSS.get(rsHash).get(0)); Long newRSAccession = this.clusteredVariantAccessioningService.getOrCreate( - Collections.singletonList(clusteredVariantEntity), jobExecution.getJobId().toString()) + Collections.singletonList(clusteredVariantEntity), jobExecution.getJobId().toString()) .get(0).getAccession(); ClusteringWriter.writeRSReportEntry(this.rsReportFileWriter, newRSAccession, rsHash); metricCompute.addCount(ClusteringMetric.CLUSTERED_VARIANTS_CREATED, 1); List associatedSSEntries = rsHashAndAssociatedSS.get(rsHash); - for (SubmittedVariantEntity submittedVariantEntity: associatedSSEntries) { - Long oldRSAccession = submittedVariantEntity.getClusteredVariantAccession(); + for (SubmittedVariantEntity submittedVariantEntity : associatedSSEntries) { + Long oldRSAccession = submittedVariantEntity.getClusteredVariantAccession(); logger.info("RS split operation: Associating ss{} with hash {} to newly issued rs{}...", - submittedVariantEntity.getAccession(), submittedVariantEntity.getHashedMessage(), - newRSAccession); + submittedVariantEntity.getAccession(), submittedVariantEntity.getHashedMessage(), + newRSAccession); associateNewRSToSS(newRSAccession, submittedVariantEntity); writeRSUpdateOperation(oldRSAccession, newRSAccession, - clusteringWriter.toClusteredVariantEntity(submittedVariantEntity)); + clusteringWriter.toClusteredVariantEntity(submittedVariantEntity)); writeSSUpdateOperation(oldRSAccession, newRSAccession, submittedVariantEntity); } } @@ -229,7 +226,7 @@ private boolean doesSSAlreadyHaveANewRS(SubmittedVariantEntity ssMarkedToReceive if (ssInDBOption.isPresent()) { ISubmittedVariant ssInDB = ssInDBOption.get(); return !(ssInDB.getClusteredVariantAccession() - .equals(ssMarkedToReceiveNewRS.getClusteredVariantAccession())); + .equals(ssMarkedToReceiveNewRS.getClusteredVariantAccession())); } return false; } @@ -270,12 +267,12 @@ private void writeRSUpdateOperation(Long oldRSAccession, Long newRSAccession, String splitOperationDescription = "Due to hash mismatch, rs" + newRSAccession + " was issued to split from rs" + oldRSAccession + "."; splitOperation.fill(EventType.RS_SPLIT, oldRSAccession, newRSAccession, splitOperationDescription, - Collections.singletonList(new ClusteredVariantInactiveEntity(clusteredVariantEntity))); + Collections.singletonList(new ClusteredVariantInactiveEntity(clusteredVariantEntity))); Query queryToCheckPreviousRSOperation = query(where(ACCESSION_ATTRIBUTE).is(oldRSAccession)) .addCriteria(where(EVENT_TYPE_ATTRIBUTE).is(EventType.RS_SPLIT)) .addCriteria(where(SPLIT_INTO_ATTRIBUTE).is(newRSAccession)) .addCriteria(where(ASSEMBLY_ATTRIBUTE_IN_CLUSTERED_OPERATIONS_COLLECTION) - .is(clusteredVariantEntity.getAssemblyAccession())); + .is(clusteredVariantEntity.getAssemblyAccession())); if (this.mongoTemplate.find(queryToCheckPreviousRSOperation, operationClass).isEmpty()) { this.mongoTemplate.insert(splitOperation, this.mongoTemplate.getCollectionName(operationClass)); metricCompute.addCount(ClusteringMetric.CLUSTERED_VARIANTS_RS_SPLIT, 1); @@ -292,13 +289,14 @@ private void writeSSUpdateOperation(Long oldRSAccession, Long newRSAccession, String updateOperationDescription = "SS was associated with the split RS rs" + newRSAccession + " that was split from rs" + oldRSAccession + " after remapping."; updateOperation.fill(EventType.UPDATED, submittedVariantEntity.getAccession(), updateOperationDescription, - Collections.singletonList(new SubmittedVariantInactiveEntity(submittedVariantEntity))); + Collections.singletonList(new SubmittedVariantInactiveEntity(submittedVariantEntity))); this.mongoTemplate.insert(updateOperation, this.mongoTemplate.getCollectionName(operationClass)); metricCompute.addCount(ClusteringMetric.SUBMITTED_VARIANTS_UPDATE_OPERATIONS, 1); } /** * Get hashes that should receive the new RS + * * @param splitCandidates List of triples * 1) ClusteredVariant object with the hash 2) the hash itself and * 3) the number of variants with that hash @@ -316,15 +314,15 @@ private List getHashesThatShouldGetNewRS(List hashThatShouldRetainOldRS.getClusteredVariantEntity().getAccession()); String hashToFind = hashThatShouldRetainOldRS.getRsHash(); Class otherCollection = - (rsCollectionToUse == DbsnpClusteredVariantEntity.class)? - ClusteredVariantEntity.class: DbsnpClusteredVariantEntity.class; + (rsCollectionToUse == DbsnpClusteredVariantEntity.class) ? + ClusteredVariantEntity.class : DbsnpClusteredVariantEntity.class; if (this.mongoTemplate.find(query(where(ID_ATTRIBUTE).is(hashToFind)), rsCollectionToUse).isEmpty()) { // Don't insert an entry with an RS hash if that hash already exists in another collection // This way, we at least make it possible for a clustering re-run to resolve a previously unresolved merge // See https://docs.google.com/spreadsheets/d/10NqzlcmKF5cVotJtnYhEFYvD_A7P9WsjN2p2ipYFKxE/edit#gid=0 // If we allow multiple hashes to co-exist in different collections, a re-run will be hindered // (see EVA-3132 and https://ebi-eva.slack.com/archives/C5A6MLDAR/p1676986532496859) - if (this.mongoTemplate.find(query(where(ID_ATTRIBUTE).is(hashToFind)),otherCollection).isEmpty()) { + if (this.mongoTemplate.find(query(where(ID_ATTRIBUTE).is(hashToFind)), otherCollection).isEmpty()) { this.mongoTemplate.insert(hashThatShouldRetainOldRS.getClusteredVariantEntity(), this.mongoTemplate.getCollectionName(rsCollectionToUse)); metricCompute.addCount(ClusteringMetric.CLUSTERED_VARIANTS_CREATED, 1); @@ -332,13 +330,13 @@ private List getHashesThatShouldGetNewRS(List logger.warn("Skipping creation of RS record " + hashThatShouldRetainOldRS.getClusteredVariantEntity() + " because the hash " + hashToFind + " is present in the " + otherCollection.getSimpleName() + " collection! This should NOT happen and likely indicates an RS merge that did not take place!" - +" See https://docs.google.com/spreadsheets/d/10NqzlcmKF5cVotJtnYhEFYvD_A7P9WsjN2p2ipYFKxE/edit#gid=0."); + + " See https://docs.google.com/spreadsheets/d/10NqzlcmKF5cVotJtnYhEFYvD_A7P9WsjN2p2ipYFKxE/edit#gid=0."); } } return splitCandidates.stream() - .map(SplitDeterminants::getRsHash) - .filter(rsHash -> !(rsHash.equals(hashThatShouldRetainOldRS.getRsHash()))) - .collect(Collectors.toList()); + .map(SplitDeterminants::getRsHash) + .filter(rsHash -> !(rsHash.equals(hashThatShouldRetainOldRS.getRsHash()))) + .collect(Collectors.toList()); } public void setJobExecution(JobExecution jobExecution) { diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/io/SSSplitWriter.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/io/SSSplitWriter.java index c8ce136f1..0eac74c2a 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/io/SSSplitWriter.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/io/SSSplitWriter.java @@ -16,20 +16,19 @@ package uk.ac.ebi.eva.accession.clustering.batch.io; import com.mongodb.MongoBulkWriteException; - +import jakarta.annotation.Nonnull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.batch.core.JobExecution; +import org.springframework.batch.item.Chunk; import org.springframework.batch.item.ItemWriter; import org.springframework.data.mongodb.core.MongoTemplate; - import org.springframework.data.mongodb.core.query.Query; import uk.ac.ebi.ampt2d.commons.accession.core.exceptions.AccessionCouldNotBeGeneratedException; import uk.ac.ebi.ampt2d.commons.accession.core.models.AccessionWrapper; import uk.ac.ebi.ampt2d.commons.accession.core.models.EventType; import uk.ac.ebi.ampt2d.commons.accession.core.models.GetOrCreateAccessionWrapper; import uk.ac.ebi.ampt2d.commons.accession.persistence.mongodb.document.EventDocument; - import uk.ac.ebi.eva.accession.clustering.batch.io.SubmittedVariantSplittingPolicy.SplitDeterminants; import uk.ac.ebi.eva.accession.clustering.metric.ClusteringMetric; import uk.ac.ebi.eva.accession.core.model.ISubmittedVariant; @@ -41,7 +40,6 @@ import uk.ac.ebi.eva.accession.core.service.nonhuman.SubmittedVariantAccessioningService; import uk.ac.ebi.eva.metrics.metric.MetricCompute; -import javax.annotation.Nonnull; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -95,11 +93,11 @@ public SSSplitWriter(String assembly, ClusteringWriter clusteringWriter, * Providing duplicates is harmless but may make the writer inefficient * since it will result in extra queries to the database. */ - public void write(@Nonnull List submittedVariantEntities) + public void write(@Nonnull Chunk submittedVariantEntities) throws MongoBulkWriteException, AccessionCouldNotBeGeneratedException, InstantiationException, IllegalAccessException { // We do this to avoid tedious generic type signatures in the method calls - List svesWithDuplicateID = new ArrayList<>(submittedVariantEntities); + List svesWithDuplicateID = new ArrayList<>(submittedVariantEntities.getItems()); // All the following steps are designed to be idempotent registerSplitCandidates(svesWithDuplicateID); @@ -110,19 +108,19 @@ public void write(@Nonnull List submittedVaria protected void processSplitCandidates(List splitCandidateOperations) throws AccessionCouldNotBeGeneratedException { Map svesToCreateWithNewIDs = new HashMap<>(); - for (SubmittedVariantOperationEntity operation: splitCandidateOperations) { + for (SubmittedVariantOperationEntity operation : splitCandidateOperations) { List splitCandidates = operation.getInactiveObjects() - .stream().map(svie -> new SplitDeterminants(svie.toSubmittedVariantEntity(), - svie.getHashedMessage(), - svie.getClusteredVariantAccession())) - .collect(Collectors.toList()); + .stream().map(svie -> new SplitDeterminants(svie.toSubmittedVariantEntity(), + svie.getHashedMessage(), + svie.getClusteredVariantAccession())) + .collect(Collectors.toList()); // Based on the split policy, one of the SS hashes will retain the SS associated with it // and the other hashes should be associated with new SS IDs - List svesThatShouldGetNewIDs = getSVEsThatShouldGetNewIDs(splitCandidates); + List svesThatShouldGetNewIDs = getSVEsThatShouldGetNewIDs(splitCandidates); - for(SplitDeterminants splitCandidate: splitCandidates) { - if(svesThatShouldGetNewIDs.contains(splitCandidate.getSubmittedVariantEntity())) { + for (SplitDeterminants splitCandidate : splitCandidates) { + if (svesThatShouldGetNewIDs.contains(splitCandidate.getSubmittedVariantEntity())) { svesToCreateWithNewIDs.put(splitCandidate.getSSHash(), splitCandidate.getSubmittedVariantEntity()); } } @@ -141,14 +139,13 @@ private void excludeSSWithAlreadyUpdatedIDs(Map List> existingSSList = this.submittedVariantAccessioningService.get(new ArrayList<>(ssHashesAndAssociatedSS.values())); - for(AccessionWrapper existingSSEntryInDB: existingSSList) { + for (AccessionWrapper existingSSEntryInDB : existingSSList) { // If some existing SS in the database have already been provided an updated accession // when processing a split operation previously, remove such SS from being considered for split String existingSSEntryInDBHash = existingSSEntryInDB.getHash(); - if(ssHashesAndAssociatedSS.containsKey(existingSSEntryInDBHash) && + if (ssHashesAndAssociatedSS.containsKey(existingSSEntryInDBHash) && !existingSSEntryInDB.getAccession().equals(ssHashesAndAssociatedSS - .get(existingSSEntryInDBHash).getAccession())) - { + .get(existingSSEntryInDBHash).getAccession())) { ssHashesAndAssociatedSS.remove(existingSSEntryInDB.getHash()); } } @@ -168,34 +165,33 @@ protected void removeCurrentSSEntriesInDBForSplitCandidates(Set ssHashes Map> dbsnpSvoeOpsToWrite = new HashMap<>(); - for(GetOrCreateAccessionWrapper newlyCreatedSVE: newlyCreatedSVEs) { + for (GetOrCreateAccessionWrapper newlyCreatedSVE : newlyCreatedSVEs) { SubmittedVariantEntity oldSVE = oldSVEs.get(newlyCreatedSVE.getHash()); String idForSplitOperation = String.format("SS_SPLIT_FROM_%s_TO_%s", oldSVE.getAccession(), - newlyCreatedSVE.getAccession()); + newlyCreatedSVE.getAccession()); logger.info("Processed split operation " + idForSplitOperation + "..."); SubmittedVariantOperationEntity splitOperation = new SubmittedVariantOperationEntity(); splitOperation.fill(EventType.SS_SPLIT, - oldSVE.getAccession(), newlyCreatedSVE.getAccession(), - String.format("SS split from %s to %s", oldSVE.getAccession(), - newlyCreatedSVE.getAccession()) + - " due to same SS ID being assigned to different SS loci", - Collections.singletonList(new SubmittedVariantInactiveEntity(oldSVE))); + oldSVE.getAccession(), newlyCreatedSVE.getAccession(), + String.format("SS split from %s to %s", oldSVE.getAccession(), + newlyCreatedSVE.getAccession()) + + " due to same SS ID being assigned to different SS loci", + Collections.singletonList(new SubmittedVariantInactiveEntity(oldSVE))); splitOperation.setId(idForSplitOperation); if (this.clusteringWriter.isEvaSubmittedVariant(oldSVE)) { svoeOpsToWrite.put(idForSplitOperation, splitOperation); - } - else { + } else { dbsnpSvoeOpsToWrite.put(idForSplitOperation, splitOperation); } } // Remove entries for which operations have already been recorded this.mongoTemplate.find(query(where(ID_ATTRIBUTE).in(svoeOpsToWrite.keySet())), - SubmittedVariantOperationEntity.class) - .forEach(existingSvoeOp -> svoeOpsToWrite.remove(existingSvoeOp.getId())); + SubmittedVariantOperationEntity.class) + .forEach(existingSvoeOp -> svoeOpsToWrite.remove(existingSvoeOp.getId())); this.mongoTemplate.find(query(where(ID_ATTRIBUTE).in(dbsnpSvoeOpsToWrite.keySet())), - DbsnpSubmittedVariantOperationEntity.class) - .forEach(existingDbsnpSvoeOp -> dbsnpSvoeOpsToWrite.remove(existingDbsnpSvoeOp.getId())); + DbsnpSubmittedVariantOperationEntity.class) + .forEach(existingDbsnpSvoeOp -> dbsnpSvoeOpsToWrite.remove(existingDbsnpSvoeOp.getId())); if (svoeOpsToWrite.values().size() > 0) { this.mongoTemplate.insert(svoeOpsToWrite.values(), SubmittedVariantOperationEntity.class); @@ -209,7 +205,7 @@ protected void removeSplitCandidates(List split List splitCandidateIDsToFind = splitCandidatesForCurrentBatch.stream().map( SubmittedVariantOperationEntity::getId).collect(Collectors.toList()); this.mongoTemplate.remove(query(where(ID_ATTRIBUTE).in(splitCandidateIDsToFind)), - SubmittedVariantOperationEntity.class); + SubmittedVariantOperationEntity.class); } protected void registerSplitCandidates(List svesWithDuplicateID) { @@ -225,7 +221,7 @@ protected void registerSplitCandidates(List svesWithDupl this.allSplitCandidatesForCurrentBatch.addAll(existingSplitCandidateOperationsInDB); Set existingSplitCandidateOperationIdsInDB = existingSplitCandidateOperationsInDB.stream().map(SubmittedVariantOperationEntity::getId) - .collect(Collectors.toSet()); + .collect(Collectors.toSet()); // Remove operations that are already in DB idsForSplitCandidateOperationsToWrite.removeAll(existingSplitCandidateOperationIdsInDB); // Bulk write split candidates for current batch @@ -236,7 +232,7 @@ private Map> getIdSVEMapForSplitCandidateOp List svesWithDuplicateID) { Map> idSVEMapForSplitCandidateOperations = new HashMap<>(); List duplicateSSIDs = svesWithDuplicateID.stream().map(SubmittedVariantEntity::getAccession) - .collect(Collectors.toList()); + .collect(Collectors.toList()); // The input list that we get may only have one of the SS entries that share duplicate IDs // We have to fetch the actual duplicates themselves in order to construct split candidates @@ -245,20 +241,19 @@ private Map> getIdSVEMapForSplitCandidateOp .getAllActiveByAssemblyAndAccessionIn(this.assembly, duplicateSSIDs) .stream() .map(result -> new SubmittedVariantEntity(result.getAccession(), result.getHash(), - result.getData(), result.getVersion())) + result.getData(), result.getVersion())) .filter(sve -> !this.variantHasMultiMapOrMismatchedAlleles(sve) && Objects.isNull(sve.getRemappedFrom())) .collect(Collectors.toList()); - for (SubmittedVariantEntity sve: svesWithDuplicateIDAlongWithActualDuplicates) { + for (SubmittedVariantEntity sve : svesWithDuplicateIDAlongWithActualDuplicates) { String idForSplitCandidateOperation = String.format("SS_SPLIT_CANDIDATES_%s_%s", - sve.getReferenceSequenceAccession(), - sve.getAccession()); + sve.getReferenceSequenceAccession(), + sve.getAccession()); if (!idSVEMapForSplitCandidateOperations.containsKey(idForSplitCandidateOperation)) { idSVEMapForSplitCandidateOperations.put(idForSplitCandidateOperation, - new ArrayList<>(Collections.singletonList(sve))); - } - else { + new ArrayList<>(Collections.singletonList(sve))); + } else { idSVEMapForSplitCandidateOperations.get(idForSplitCandidateOperation).add(sve); } } @@ -269,7 +264,7 @@ private void writeSplitCandidatesForCurrentBatchToDB(Map idsForSplitCandidateOperationsToWrite) { // Only new split candidates in the current batch should be written to the database List newSplitCandidatesForCurrentBatch = new ArrayList<>(); - for (String id: idsForSplitCandidateOperationsToWrite) { + for (String id : idsForSplitCandidateOperationsToWrite) { SubmittedVariantOperationEntity splitCandidateOperation = new SubmittedVariantOperationEntity(); List svesThatShareSameID = idSVEMap.get(id); // Ensure that non-duplicate SS IDs are not processed @@ -278,9 +273,9 @@ private void writeSplitCandidatesForCurrentBatchToDB(Map> { diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/io/qc/DuplicateRSAccQCWriter.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/io/qc/DuplicateRSAccQCWriter.java index 3a8b32026..321929bcd 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/io/qc/DuplicateRSAccQCWriter.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/io/qc/DuplicateRSAccQCWriter.java @@ -1,9 +1,10 @@ package uk.ac.ebi.eva.accession.clustering.batch.io.qc; -import gherkin.deps.com.google.gson.Gson; -import gherkin.deps.com.google.gson.GsonBuilder; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.batch.item.Chunk; import org.springframework.batch.item.ExecutionContext; import org.springframework.batch.item.ItemStreamException; import org.springframework.batch.item.ItemStreamWriter; @@ -48,7 +49,7 @@ public void close() throws ItemStreamException { } @Override - public void write(List> listOfDuplicateRSAccQCResultLists) throws Exception { + public void write(Chunk> listOfDuplicateRSAccQCResultLists) throws Exception { for (List duplicateRSAccQCResultList : listOfDuplicateRSAccQCResultLists) { if (duplicateRSAccQCResultList != null && !duplicateRSAccQCResultList.isEmpty()) { appendToFile(duplicateRSAccQCResultList); diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/io/qc/ExtraneousRSReporter.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/io/qc/ExtraneousRSReporter.java index 5b78627bd..9a0caf252 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/io/qc/ExtraneousRSReporter.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/io/qc/ExtraneousRSReporter.java @@ -16,9 +16,10 @@ package uk.ac.ebi.eva.accession.clustering.batch.io.qc; import com.mongodb.MongoBulkWriteException; -import org.apache.commons.collections.CollectionUtils; +import jakarta.annotation.Nonnull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.batch.item.Chunk; import org.springframework.batch.item.ItemWriter; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.data.mongodb.core.query.Criteria; @@ -27,9 +28,8 @@ import uk.ac.ebi.eva.accession.clustering.batch.io.qc.QCMongoCollections.qcRSIdInSS; import uk.ac.ebi.eva.accession.core.model.eva.ClusteredVariantEntity; -import javax.annotation.Nonnull; -import java.util.Arrays; import java.util.List; +import java.util.Set; import java.util.stream.Collectors; import static org.springframework.data.mongodb.core.query.Criteria.where; @@ -50,24 +50,24 @@ public ExtraneousRSReporter(String assemblyAccession, MongoTemplate mongoTemplat } @Override - public void write(@Nonnull List clusteredVariantEntities) + public void write(@Nonnull Chunk clusteredVariantEntities) throws MongoBulkWriteException, AccessionCouldNotBeGeneratedException { reportExtraneousRS(clusteredVariantEntities); } - private void reportExtraneousRS(List clusteredVariantEntities) { + private void reportExtraneousRS(Chunk clusteredVariantEntities) { String assemblyAccessionPrefix = QCMongoCollections.getAssemblyAccessionPrefix(this.assemblyAccession); - List idsFromRSIDCollection = clusteredVariantEntities.stream().map( + List idsFromRSIDCollection = clusteredVariantEntities.getItems().stream().map( entity -> assemblyAccessionPrefix + entity.getAccession()).distinct().collect(Collectors.toList()); - Criteria[] criteriaToLookupIDs = + Criteria[] criteriaToLookupIDs = idsFromRSIDCollection.stream().map(id -> where(IDAttribute).is(id)).toArray(Criteria[]::new); if (criteriaToLookupIDs.length == 0) return; Query queryToLookupIDs = new Query(new Criteria().orOperator(criteriaToLookupIDs)); - List idsInSSIDCollection = this.mongoTemplate.find(queryToLookupIDs, qcRSIdInSS.class) - .stream().map(qcRSIdInSS::getId).collect(Collectors.toList()); + Set idsInSSIDCollection = this.mongoTemplate.find(queryToLookupIDs, qcRSIdInSS.class) + .stream().map(qcRSIdInSS::getId).collect(Collectors.toSet()); - Arrays.stream(CollectionUtils.subtract(idsFromRSIDCollection, idsInSSIDCollection).toArray()) - .map(Object::toString) + idsFromRSIDCollection.stream() + .filter(id -> !idsInSSIDCollection.contains(id)) .forEach(extraneousRS -> logger.error("RS ID rs{} was not assigned to any SS in the assembly {}", extraneousRS.replace(assemblyAccessionPrefix, ""), this.assemblyAccession)); diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/io/qc/MissingCveReporter.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/io/qc/MissingCveReporter.java index 470902a6a..4d1ea4e64 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/io/qc/MissingCveReporter.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/io/qc/MissingCveReporter.java @@ -15,16 +15,16 @@ */ package uk.ac.ebi.eva.accession.clustering.batch.io.qc; +import jakarta.annotation.Nonnull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.batch.item.Chunk; import org.springframework.batch.item.ItemWriter; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.data.mongodb.core.query.Query; - import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpClusteredVariantEntity; import uk.ac.ebi.eva.accession.core.model.eva.ClusteredVariantEntity; -import javax.annotation.Nonnull; import java.util.List; import java.util.Map; import java.util.Objects; @@ -48,9 +48,9 @@ public MissingCveReporter(MongoTemplate mongoTemplate) { } @Override - public void write(@Nonnull List rsHashPairs) { - Map hashToRs = rsHashPairs.stream() - .collect(Collectors.toMap(RSHashPair::getHash, RSHashPair::getRsId)); + public void write(@Nonnull Chunk rsHashPairs) { + Map hashToRs = rsHashPairs.getItems().stream() + .collect(Collectors.toMap(RSHashPair::getHash, RSHashPair::getRsId)); Map results = findClusteredVariantsInDb(hashToRs); for (String hash : hashToRs.keySet()) { @@ -71,7 +71,7 @@ private Map findClusteredVariantsInDb(Map evaResults = mongoTemplate.find(query, ClusteredVariantEntity.class); List dbsnpResults = mongoTemplate.find(query, DbsnpClusteredVariantEntity.class); return Stream.concat(evaResults.stream(), dbsnpResults.stream()) - .collect(Collectors.toMap(ClusteredVariantEntity::getHashedMessage, Function.identity())); + .collect(Collectors.toMap(ClusteredVariantEntity::getHashedMessage, Function.identity())); } } diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/io/qc/PendingMergeSplitReporter.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/io/qc/PendingMergeSplitReporter.java index 628adf5c6..4cb1cc843 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/io/qc/PendingMergeSplitReporter.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/io/qc/PendingMergeSplitReporter.java @@ -16,21 +16,23 @@ package uk.ac.ebi.eva.accession.clustering.batch.io.qc; import com.mongodb.MongoBulkWriteException; +import jakarta.annotation.Nonnull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.batch.item.Chunk; import org.springframework.batch.item.ItemWriter; import org.springframework.dao.DuplicateKeyException; import org.springframework.data.mongodb.core.BulkOperations; import org.springframework.data.mongodb.core.MongoTemplate; import uk.ac.ebi.ampt2d.commons.accession.core.exceptions.AccessionCouldNotBeGeneratedException; - import uk.ac.ebi.eva.accession.clustering.batch.io.ClusteringWriter; import uk.ac.ebi.eva.accession.clustering.batch.io.qc.QCMongoCollections.qcRSHashInSS; import uk.ac.ebi.eva.accession.clustering.batch.io.qc.QCMongoCollections.qcRSIdInSS; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; -import javax.annotation.Nonnull; -import java.util.*; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; import java.util.stream.Collectors; import static org.springframework.data.mongodb.core.query.Criteria.where; @@ -57,18 +59,18 @@ public PendingMergeSplitReporter(String assemblyAccession, ClusteringWriter clus } @Override - public void write(@Nonnull List submittedVariantEntities) + public void write(@Nonnull Chunk submittedVariantEntities) throws MongoBulkWriteException, AccessionCouldNotBeGeneratedException { if (submittedVariantEntities.size() > 0) { reportPendingSplitsAndMerges(submittedVariantEntities); } } - private void reportPendingSplitsAndMerges(List submittedVariantEntities) { + private void reportPendingSplitsAndMerges(Chunk submittedVariantEntities) { Map hashAndAssociatedRS = new HashMap<>(); Map rsAndAssociatedHash = new HashMap<>(); String assemblyAccessionPrefix = QCMongoCollections.getAssemblyAccessionPrefix(this.assemblyAccession); - for (SubmittedVariantEntity submittedVariantEntity :submittedVariantEntities) { + for (SubmittedVariantEntity submittedVariantEntity : submittedVariantEntities) { Long rsID = submittedVariantEntity.getClusteredVariantAccession(); if (Objects.nonNull(rsID)) { String rsHash = clusteringWriter.toClusteredVariantEntity(submittedVariantEntity).getHashedMessage(); @@ -90,33 +92,32 @@ private void reportPendingSplitsAndMerges(List if (hashAndAssociatedRS.size() > 0) { bulkHashInsert.execute(); } - } - catch (DuplicateKeyException duplicateKeyException) { + } catch (DuplicateKeyException duplicateKeyException) { MongoBulkWriteException writeException = ((MongoBulkWriteException) duplicateKeyException.getCause()); for (String hashWithAssemblyPrefix : extractUniqueHashesForDuplicateKeyError(writeException).collect( Collectors.toList())) { qcRSHashInSS result = this.mongoTemplate.findOne(query(where(IDAttribute).is(hashWithAssemblyPrefix)), - qcRSHashInSS.class); - reportMultipleRSWithSameHash (hashAndAssociatedRS, assemblyAccessionPrefix, result.getRsID(), - hashWithAssemblyPrefix.replace(assemblyAccessionPrefix, "")); - }; + qcRSHashInSS.class); + reportMultipleRSWithSameHash(hashAndAssociatedRS, assemblyAccessionPrefix, result.getRsID(), + hashWithAssemblyPrefix.replace(assemblyAccessionPrefix, "")); + } + ; } try { if (rsAndAssociatedHash.size() > 0) { bulkIDInsert.execute(); } - } - catch (DuplicateKeyException duplicateKeyException) { + } catch (DuplicateKeyException duplicateKeyException) { MongoBulkWriteException writeException = ((MongoBulkWriteException) duplicateKeyException.getCause()); for (String idWithAssemblyPrefix : extractUniqueHashesForDuplicateKeyError(writeException).collect( Collectors.toList())) { qcRSIdInSS result = this.mongoTemplate.findOne(query(where(IDAttribute).is(idWithAssemblyPrefix)), - qcRSIdInSS.class); + qcRSIdInSS.class); reportSameRSWithMultipleHashes(rsAndAssociatedHash, assemblyAccessionPrefix, - Long.parseLong(idWithAssemblyPrefix.replace(assemblyAccessionPrefix, "")), - result.getHash()); - }; + Long.parseLong(idWithAssemblyPrefix.replace(assemblyAccessionPrefix, "")), + result.getHash()); + } } } diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/io/qc/RSReader.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/io/qc/RSReader.java index c1eb7e253..2576aee8a 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/io/qc/RSReader.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/io/qc/RSReader.java @@ -19,6 +19,7 @@ import com.mongodb.client.FindIterable; import com.mongodb.client.MongoCursor; import com.mongodb.client.model.Filters; +import jakarta.annotation.Nonnull; import org.bson.Document; import org.bson.conversions.Bson; import org.slf4j.Logger; @@ -31,8 +32,6 @@ import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpClusteredVariantEntity; import uk.ac.ebi.eva.accession.core.model.eva.ClusteredVariantEntity; -import javax.annotation.Nonnull; - public class RSReader implements ItemStreamReader { @@ -79,7 +78,7 @@ public void open(@Nonnull ExecutionContext executionContext) throws ItemStreamEx public void initializeReader() { Bson query = Filters.and(Filters.in(ASSEMBLY_FIELD, assembly), - Filters.not(Filters.exists(MAPPING_WEIGHT_FIELD))); + Filters.not(Filters.exists(MAPPING_WEIGHT_FIELD))); logger.info("Issuing find: {}", query); FindIterable clusteredVariantsDbsnp = @@ -94,9 +93,9 @@ public void initializeReader() { private FindIterable getClusteredVariants(Bson query, Class entityClass) { return mongoTemplate.getCollection(mongoTemplate.getCollectionName(entityClass)) - .find(query) - .noCursorTimeout(true) - .batchSize(chunkSize); + .find(query) + .noCursorTimeout(true) + .batchSize(chunkSize); } @Override diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/io/qc/SSReader.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/io/qc/SSReader.java index fda379834..eb4305014 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/io/qc/SSReader.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/io/qc/SSReader.java @@ -19,6 +19,7 @@ import com.mongodb.client.FindIterable; import com.mongodb.client.MongoCursor; import com.mongodb.client.model.Filters; +import jakarta.annotation.Nonnull; import org.bson.Document; import org.bson.conversions.Bson; import org.slf4j.Logger; @@ -28,12 +29,9 @@ import org.springframework.batch.item.ItemStreamReader; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.data.mongodb.core.convert.MongoConverter; - import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpSubmittedVariantEntity; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; -import javax.annotation.Nonnull; - public class SSReader implements ItemStreamReader { @@ -80,7 +78,7 @@ public void open(ExecutionContext executionContext) throws ItemStreamException { public void initializeReader() { Bson query = Filters.and(Filters.in(ASSEMBLY_FIELD, assembly), - Filters.not(Filters.exists(MAPPING_WEIGHT_FIELD))); + Filters.not(Filters.exists(MAPPING_WEIGHT_FIELD))); logger.info("Issuing find: {}", query); FindIterable submittedVariantsDbsnp = @@ -95,9 +93,9 @@ public void initializeReader() { private FindIterable getSubmittedVariants(Bson query, Class entityClass) { return mongoTemplate.getCollection(mongoTemplate.getCollectionName(entityClass)) - .find(query) - .noCursorTimeout(true) - .batchSize(chunkSize); + .find(query) + .noCursorTimeout(true) + .batchSize(chunkSize); } @Override diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/processors/VariantToSubmittedVariantEntityProcessor.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/processors/VariantToSubmittedVariantEntityProcessor.java index 5f0032d07..ef1b7c376 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/processors/VariantToSubmittedVariantEntityProcessor.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/processors/VariantToSubmittedVariantEntityProcessor.java @@ -18,15 +18,14 @@ import org.springframework.batch.item.ItemProcessor; import org.springframework.util.Assert; import uk.ac.ebi.ampt2d.commons.accession.hashing.SHA1HashingFunction; - import uk.ac.ebi.eva.accession.core.model.ISubmittedVariant; import uk.ac.ebi.eva.accession.core.model.SubmittedVariant; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; import uk.ac.ebi.eva.accession.core.summary.SubmittedVariantSummaryFunction; -import uk.ac.ebi.eva.remapping.source.batch.io.VariantContextWriter; -import uk.ac.ebi.eva.remapping.source.batch.processors.SubmittedVariantToVariantContextProcessor; import uk.ac.ebi.eva.commons.core.models.pipeline.Variant; import uk.ac.ebi.eva.commons.core.models.pipeline.VariantSourceEntry; +import uk.ac.ebi.eva.remapping.source.batch.io.VariantContextWriter; +import uk.ac.ebi.eva.remapping.source.batch.processors.SubmittedVariantToVariantContextProcessor; import java.util.function.Function; import java.util.stream.Collectors; diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/RunnerConfiguration.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/RunnerConfiguration.java index d17811c9f..f3f598c7b 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/RunnerConfiguration.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/RunnerConfiguration.java @@ -16,15 +16,10 @@ */ package uk.ac.ebi.eva.accession.clustering.configuration.batch; -import org.springframework.batch.core.configuration.annotation.BatchConfigurer; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import uk.ac.ebi.eva.commons.batch.configuration.SpringBoot1CompatibilityConfiguration; import uk.ac.ebi.eva.commons.batch.job.JobExecutionApplicationListener; -import javax.persistence.EntityManagerFactory; -import javax.sql.DataSource; - @Configuration public class RunnerConfiguration { @@ -32,11 +27,4 @@ public class RunnerConfiguration { public JobExecutionApplicationListener jobExecutionApplicationListener() { return new JobExecutionApplicationListener(); } - - @Bean - public BatchConfigurer configurer(DataSource dataSource, EntityManagerFactory entityManagerFactory) - throws Exception { - return SpringBoot1CompatibilityConfiguration.getSpringBoot1CompatibleBatchConfigurer(dataSource, - entityManagerFactory); - } } diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/io/RSMergeAndSplitCandidatesReaderConfiguration.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/io/RSMergeAndSplitCandidatesReaderConfiguration.java index d009ee020..6b0ef84ee 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/io/RSMergeAndSplitCandidatesReaderConfiguration.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/io/RSMergeAndSplitCandidatesReaderConfiguration.java @@ -15,7 +15,7 @@ */ package uk.ac.ebi.eva.accession.clustering.configuration.batch.io; -import com.mongodb.DBCollection; +import org.springframework.batch.item.Chunk; import org.springframework.batch.item.ItemWriter; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -24,7 +24,6 @@ import org.springframework.data.mongodb.core.query.Criteria; import org.springframework.data.mongodb.core.query.Meta; import org.springframework.data.mongodb.core.query.Query; - import uk.ac.ebi.ampt2d.commons.accession.core.models.EventType; import uk.ac.ebi.eva.accession.clustering.configuration.InputParametersConfiguration; import uk.ac.ebi.eva.accession.clustering.parameters.InputParameters; @@ -33,7 +32,6 @@ import uk.ac.ebi.eva.commons.mongodb.readers.MongoDbCursorItemReader; import java.util.Arrays; -import java.util.List; import static org.springframework.data.mongodb.core.query.Criteria.where; import static org.springframework.data.mongodb.core.query.Query.query; @@ -60,19 +58,18 @@ public class RSMergeAndSplitCandidatesReaderConfiguration { public static final String SPLIT_CANDIDATE_ID_PREFIX = "RSSC"; public static Criteria getSplitCandidatesCriteria(String assemblyAccession) { - return where(DBCollection.ID_FIELD_NAME).regex("^" + SPLIT_CANDIDATE_ID_PREFIX + "_" - + assemblyAccession + "_.*"); + return where("_id").regex("^" + SPLIT_CANDIDATE_ID_PREFIX + "_" + + assemblyAccession + "_.*"); } public static Criteria getMergeCandidatesCriteria(String assemblyAccession) { - return where(DBCollection.ID_FIELD_NAME).regex("^" + MERGE_CANDIDATE_ID_PREFIX + "_" - + assemblyAccession + "_.*"); + return where("_id").regex("^" + MERGE_CANDIDATE_ID_PREFIX + "_" + + assemblyAccession + "_.*"); } @Bean(RS_SPLIT_CANDIDATES_READER) public MongoDbCursorItemReader rsSplitCandidatesReader(MongoTemplate mongoTemplate, - InputParameters parameters) - { + InputParameters parameters) { MongoDbCursorItemReader mongoItemReader = new MongoDbCursorItemReader<>(); mongoItemReader.setMongoTemplate(mongoTemplate); mongoItemReader.setTargetType(SubmittedVariantOperationEntity.class); @@ -90,8 +87,7 @@ public MongoDbCursorItemReader rsSplitCandidate @Bean(RS_MERGE_CANDIDATES_READER) public MongoDbCursorItemReader rsMergeCandidatesReader(MongoTemplate mongoTemplate, - InputParameters parameters) - { + InputParameters parameters) { MongoDbCursorItemReader mongoItemReader = new MongoDbCursorItemReader<>(); mongoItemReader.setMongoTemplate(mongoTemplate); mongoItemReader.setTargetType(SubmittedVariantOperationEntity.class); @@ -116,18 +112,20 @@ public MongoDbCursorItemReader rsMergeCandidate public static class NoOpItemWriter implements ItemWriter { private final MongoTemplate mongoTemplate; private final InputParameters parameters; + public NoOpItemWriter(MongoTemplate mongoTemplate, InputParameters parameters) { this.mongoTemplate = mongoTemplate; this.parameters = parameters; } + @Override - public void write(List items) throws Exception { + public void write(Chunk items) throws Exception { Query queryToRemoveMergeAndSplitCandidates = query(where(ASSEMBLY_FIELD).is(parameters.getAssemblyAccession())) - .addCriteria(where(EVENT_TYPE_FIELD).in( - Arrays.asList(MERGE_CANDIDATES_EVENT_TYPE.toString(), - SPLIT_CANDIDATES_EVENT_TYPE.toString())) - ); + .addCriteria(where(EVENT_TYPE_FIELD).in( + Arrays.asList(MERGE_CANDIDATES_EVENT_TYPE.toString(), + SPLIT_CANDIDATES_EVENT_TYPE.toString())) + ); mongoTemplate.remove(queryToRemoveMergeAndSplitCandidates, SUBMITTED_VARIANT_OPERATIONS_COLLECTION); } } diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/BackPropagateRSJobConfiguration.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/BackPropagateRSJobConfiguration.java index 3f0c1551e..ec9f999b8 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/BackPropagateRSJobConfiguration.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/BackPropagateRSJobConfiguration.java @@ -18,13 +18,17 @@ import org.springframework.batch.core.Job; import org.springframework.batch.core.Step; import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; -import org.springframework.batch.core.configuration.annotation.JobBuilderFactory; +import org.springframework.batch.core.job.builder.JobBuilder; import org.springframework.batch.core.launch.support.RunIdIncrementer; +import org.springframework.batch.core.repository.JobRepository; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.*; +import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.BACK_PROPAGATE_NEW_RS_JOB; +import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.BACK_PROPAGATE_NEW_RS_STEP; +import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.BACK_PROPAGATE_SPLIT_OR_MERGED_RS_JOB; +import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.BACK_PROPAGATE_SPLIT_OR_MERGED_RS_STEP; @Configuration @EnableBatchProcessing @@ -34,8 +38,8 @@ public class BackPropagateRSJobConfiguration { public Job backPropagateNewRSJob( // Back-propagate RS that were newly created in the remapped assembly @Qualifier(BACK_PROPAGATE_NEW_RS_STEP) Step backPropagateNewRSStep, - JobBuilderFactory jobBuilderFactory) { - return jobBuilderFactory.get(BACK_PROPAGATE_NEW_RS_JOB) + JobRepository jobRepository) { + return new JobBuilder(BACK_PROPAGATE_NEW_RS_JOB, jobRepository) .incrementer(new RunIdIncrementer()) .start(backPropagateNewRSStep) .build(); @@ -47,8 +51,8 @@ public Job backPropagateNewRSJob( public Job backPropagateSplitOrMergedRSJob( // Back-propagate RS in the remapped assembly that were split or merged @Qualifier(BACK_PROPAGATE_SPLIT_OR_MERGED_RS_STEP) Step backPropagateSplitMergedRSStep, - JobBuilderFactory jobBuilderFactory) { - return jobBuilderFactory.get(BACK_PROPAGATE_SPLIT_OR_MERGED_RS_JOB) + JobRepository jobRepository) { + return new JobBuilder(BACK_PROPAGATE_SPLIT_OR_MERGED_RS_JOB, jobRepository) .incrementer(new RunIdIncrementer()) .start(backPropagateSplitMergedRSStep) .build(); diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/ClusterUnclusteredVariantsJobConfiguration.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/ClusterUnclusteredVariantsJobConfiguration.java index 2399d090c..3740f1fd2 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/ClusterUnclusteredVariantsJobConfiguration.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/ClusterUnclusteredVariantsJobConfiguration.java @@ -19,8 +19,9 @@ import org.springframework.batch.core.JobExecutionListener; import org.springframework.batch.core.Step; import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; -import org.springframework.batch.core.configuration.annotation.JobBuilderFactory; +import org.springframework.batch.core.job.builder.JobBuilder; import org.springframework.batch.core.launch.support.RunIdIncrementer; +import org.springframework.batch.core.repository.JobRepository; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -47,15 +48,15 @@ public Job clusteringFromMongoJob( @Qualifier(CLUSTERING_NON_CLUSTERED_VARIANTS_FROM_MONGO_STEP) Step clusteringNonClusteredVariantsFromMongoStep, @Qualifier(ACCESSIONING_SHUTDOWN_STEP) Step accessioningShutdownStep, @Qualifier(JOB_EXECUTION_LISTENER) JobExecutionListener jobExecutionListener, - JobBuilderFactory jobBuilderFactory) { - return jobBuilderFactory.get(CLUSTER_UNCLUSTERED_VARIANTS_JOB) - .incrementer(new RunIdIncrementer()) - .start(processRSMergeCandidatesStep) - .next(processRSSplitCandidatesStep) - .next(clearRSMergeAndSplitCandidatesStep) - .next(clusteringNonClusteredVariantsFromMongoStep) - .next(accessioningShutdownStep) - .listener(jobExecutionListener) - .build(); + JobRepository jobRepository) { + return new JobBuilder(CLUSTER_UNCLUSTERED_VARIANTS_JOB, jobRepository) + .incrementer(new RunIdIncrementer()) + .start(processRSMergeCandidatesStep) + .next(processRSSplitCandidatesStep) + .next(clearRSMergeAndSplitCandidatesStep) + .next(clusteringNonClusteredVariantsFromMongoStep) + .next(accessioningShutdownStep) + .listener(jobExecutionListener) + .build(); } } diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/ClusteringFromMongoJobConfiguration.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/ClusteringFromMongoJobConfiguration.java index ea1c4a653..e4c27393d 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/ClusteringFromMongoJobConfiguration.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/ClusteringFromMongoJobConfiguration.java @@ -20,25 +20,22 @@ import org.springframework.batch.core.JobExecution; import org.springframework.batch.core.JobExecutionListener; import org.springframework.batch.core.Step; -import org.springframework.batch.core.StepContribution; import org.springframework.batch.core.StepExecution; - import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; -import org.springframework.batch.core.configuration.annotation.JobBuilderFactory; -import org.springframework.batch.core.configuration.annotation.StepBuilderFactory; import org.springframework.batch.core.job.builder.FlowBuilder; +import org.springframework.batch.core.job.builder.JobBuilder; import org.springframework.batch.core.job.flow.FlowExecutionStatus; import org.springframework.batch.core.job.flow.JobExecutionDecider; import org.springframework.batch.core.job.flow.support.SimpleFlow; import org.springframework.batch.core.launch.support.RunIdIncrementer; -import org.springframework.batch.core.scope.context.ChunkContext; -import org.springframework.batch.core.step.tasklet.Tasklet; +import org.springframework.batch.core.repository.JobRepository; +import org.springframework.batch.core.step.builder.StepBuilder; import org.springframework.batch.repeat.RepeatStatus; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.lang.NonNull; - +import org.springframework.transaction.PlatformTransactionManager; import uk.ac.ebi.eva.accession.clustering.parameters.InputParameters; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.ACCESSIONING_SHUTDOWN_STEP; @@ -60,20 +57,17 @@ private JobExecutionDecider isRemappedAssemblyPresent(InputParameters inputParam @Override @NonNull public FlowExecutionStatus decide(@NonNull JobExecution jobExecution, StepExecution stepExecution) { - String status = (!StringUtil.isBlank(inputParameters.getRemappedFrom())) ? "TRUE": "FALSE"; + String status = (!StringUtil.isBlank(inputParameters.getRemappedFrom())) ? "TRUE" : "FALSE"; return new FlowExecutionStatus(status); } }; } - private Step dummyStep(StepBuilderFactory stepBuilderFactory) { - return stepBuilderFactory.get("step_" + "dummyStep").tasklet(new Tasklet() { - @Override - public RepeatStatus execute(@NonNull StepContribution stepContribution, - @NonNull ChunkContext chunkContext) { - return RepeatStatus.FINISHED; - } - }).build(); + private Step dummyStep(JobRepository jobRepository, PlatformTransactionManager transactionManager) { + return new StepBuilder("step_" + "dummyStep", jobRepository) + .tasklet((stepContribution, chunkContext) -> RepeatStatus.FINISHED, + transactionManager) + .build(); } @Bean(CLUSTERING_FROM_MONGO_JOB) @@ -87,36 +81,35 @@ public Job clusteringFromMongoJob(@Qualifier(CLUSTERING_CLUSTERED_VARIANTS_FROM_ @Qualifier(BACK_PROPAGATE_NEW_RS_STEP) Step backPropagateNewRSStep, // Back-propagate RS in the remapped assembly that were split or merged @Qualifier(BACK_PROPAGATE_SPLIT_OR_MERGED_RS_STEP) - Step backPropagateSplitMergedRSStep, + Step backPropagateSplitMergedRSStep, @Qualifier(JOB_EXECUTION_LISTENER) JobExecutionListener jobExecutionListener, - StepBuilderFactory stepBuilderFactory, - JobBuilderFactory jobBuilderFactory, + JobRepository jobRepository, PlatformTransactionManager transactionManager, InputParameters inputParameters) { JobExecutionDecider jobExecutionDecider = isRemappedAssemblyPresent(inputParameters); - Step dummyStep = dummyStep(stepBuilderFactory); - return jobBuilderFactory.get(CLUSTERING_FROM_MONGO_JOB) + Step dummyStep = dummyStep(jobRepository, transactionManager); + return new JobBuilder(CLUSTERING_FROM_MONGO_JOB, jobRepository) .incrementer(new RunIdIncrementer()) //We need the dummy step here because Spring won't conditionally start the first step .start(dummyStep) .listener(jobExecutionListener) .next(jobExecutionDecider) - .on("TRUE") - .to(new FlowBuilder("remappedAssemblyClusteringFlow") - .start(clusteringClusteredVariantsFromMongoStep) - .next(processRSMergeCandidatesStep) - .next(processRSSplitCandidatesStep) - .next(clearRSMergeAndSplitCandidatesStep) - .next(clusteringNonClusteredVariantsFromMongoStep) - .next(accessioningShutdownStep) - .next(backPropagateNewRSStep) - .next(backPropagateSplitMergedRSStep) - .build()) - .on("*").end() + .on("TRUE") + .to(new FlowBuilder("remappedAssemblyClusteringFlow") + .start(clusteringClusteredVariantsFromMongoStep) + .next(processRSMergeCandidatesStep) + .next(processRSSplitCandidatesStep) + .next(clearRSMergeAndSplitCandidatesStep) + .next(clusteringNonClusteredVariantsFromMongoStep) + .next(accessioningShutdownStep) + .next(backPropagateNewRSStep) + .next(backPropagateSplitMergedRSStep) + .build()) + .on("*").end() .from(jobExecutionDecider) - .on("FALSE") - .to(clusteringNonClusteredVariantsFromMongoStep) - .next(accessioningShutdownStep) - .on("*").end() + .on("FALSE") + .to(clusteringNonClusteredVariantsFromMongoStep) + .next(accessioningShutdownStep) + .on("*").end() .end().build(); } } diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/ProcessRemappedVariantsWithRSJobConfiguration.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/ProcessRemappedVariantsWithRSJobConfiguration.java index 00c0922bb..6a9092b39 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/ProcessRemappedVariantsWithRSJobConfiguration.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/ProcessRemappedVariantsWithRSJobConfiguration.java @@ -18,8 +18,9 @@ import org.springframework.batch.core.Job; import org.springframework.batch.core.Step; import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; -import org.springframework.batch.core.configuration.annotation.JobBuilderFactory; +import org.springframework.batch.core.job.builder.JobBuilder; import org.springframework.batch.core.launch.support.RunIdIncrementer; +import org.springframework.batch.core.repository.JobRepository; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -35,10 +36,10 @@ public class ProcessRemappedVariantsWithRSJobConfiguration { @Bean(PROCESS_REMAPPED_VARIANTS_WITH_RS_JOB) public Job processRemappedVariantsWithRSJob( @Qualifier(CLUSTERING_CLUSTERED_VARIANTS_FROM_MONGO_STEP) Step clusteringClusteredVariantsFromMongoStep, - JobBuilderFactory jobBuilderFactory) { - return jobBuilderFactory.get(PROCESS_REMAPPED_VARIANTS_WITH_RS_JOB) - .incrementer(new RunIdIncrementer()) - .start(clusteringClusteredVariantsFromMongoStep) - .build(); + JobRepository jobRepository) { + return new JobBuilder(PROCESS_REMAPPED_VARIANTS_WITH_RS_JOB, jobRepository) + .incrementer(new RunIdIncrementer()) + .start(clusteringClusteredVariantsFromMongoStep) + .build(); } } diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/RSAccessionRecoveryJobConfiguration.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/RSAccessionRecoveryJobConfiguration.java index d9d80cafc..7070c9dbc 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/RSAccessionRecoveryJobConfiguration.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/RSAccessionRecoveryJobConfiguration.java @@ -4,8 +4,9 @@ import org.springframework.batch.core.JobExecutionListener; import org.springframework.batch.core.Step; import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; -import org.springframework.batch.core.configuration.annotation.JobBuilderFactory; +import org.springframework.batch.core.job.builder.JobBuilder; import org.springframework.batch.core.launch.support.RunIdIncrementer; +import org.springframework.batch.core.repository.JobRepository; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; @@ -28,8 +29,8 @@ public class RSAccessionRecoveryJobConfiguration { private JobExecutionListener monotonicAccessionRecoveryAgentCategoryRSJobListener; @Bean(RS_ACCESSION_RECOVERY_JOB) - public Job createMonotonicAccessionRecoveryAgentCategoryRSJob(JobBuilderFactory jobBuilderFactory) { - return jobBuilderFactory.get(RS_ACCESSION_RECOVERY_JOB) + public Job createMonotonicAccessionRecoveryAgentCategoryRSJob(JobRepository jobRepository) { + return new JobBuilder(RS_ACCESSION_RECOVERY_JOB, jobRepository) .incrementer(new RunIdIncrementer()) .start(monotonicAccessionRecoveryAgentCategoryRSStep) .listener(monotonicAccessionRecoveryAgentCategoryRSJobListener) diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/StudyClusteringJobConfiguration.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/StudyClusteringJobConfiguration.java index 152e04ceb..36344d7e6 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/StudyClusteringJobConfiguration.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/StudyClusteringJobConfiguration.java @@ -19,8 +19,9 @@ import org.springframework.batch.core.JobExecutionListener; import org.springframework.batch.core.Step; import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; -import org.springframework.batch.core.configuration.annotation.JobBuilderFactory; +import org.springframework.batch.core.job.builder.JobBuilder; import org.springframework.batch.core.launch.support.RunIdIncrementer; +import org.springframework.batch.core.repository.JobRepository; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -38,12 +39,12 @@ public class StudyClusteringJobConfiguration { public Job studyClusteringJob(@Qualifier(STUDY_CLUSTERING_STEP) Step clusteringStep, @Qualifier(ACCESSIONING_SHUTDOWN_STEP) Step accessioningShutdownStep, @Qualifier(JOB_EXECUTION_LISTENER) JobExecutionListener jobExecutionListener, - JobBuilderFactory jobBuilderFactory) { - return jobBuilderFactory.get(STUDY_CLUSTERING_JOB) - .incrementer(new RunIdIncrementer()) - .start(clusteringStep) - .next(accessioningShutdownStep) - .listener(jobExecutionListener) - .build(); + JobRepository jobRepository) { + return new JobBuilder(STUDY_CLUSTERING_JOB, jobRepository) + .incrementer(new RunIdIncrementer()) + .start(clusteringStep) + .next(accessioningShutdownStep) + .listener(jobExecutionListener) + .build(); } } diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/qc/ClusteringQCJobConfiguration.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/qc/ClusteringQCJobConfiguration.java index 4308574bc..69717ddfd 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/qc/ClusteringQCJobConfiguration.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/qc/ClusteringQCJobConfiguration.java @@ -19,9 +19,10 @@ import org.springframework.batch.core.Step; import org.springframework.batch.core.StepExecutionListener; import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; -import org.springframework.batch.core.configuration.annotation.JobBuilderFactory; -import org.springframework.batch.core.configuration.annotation.StepBuilderFactory; +import org.springframework.batch.core.job.builder.JobBuilder; import org.springframework.batch.core.launch.support.RunIdIncrementer; +import org.springframework.batch.core.repository.JobRepository; +import org.springframework.batch.core.step.builder.StepBuilder; import org.springframework.batch.item.ItemProcessor; import org.springframework.batch.item.ItemStreamReader; import org.springframework.batch.item.ItemWriter; @@ -30,6 +31,7 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.transaction.PlatformTransactionManager; import uk.ac.ebi.eva.accession.clustering.batch.io.ClusteringWriter; import uk.ac.ebi.eva.accession.clustering.batch.io.qc.ExtraneousRSReporter; import uk.ac.ebi.eva.accession.clustering.batch.io.qc.PendingMergeSplitReporter; @@ -91,14 +93,14 @@ public ExtraneousRSReporter extraneousRSReporter(InputParameters parameters, Mon public Step reportUnclusteredSSAndPendingMergeSplitStep( @Qualifier(REMAPPED_SS_READER) ItemStreamReader remappedSSReader, @Qualifier(REPORT_UNCLUSTERED_SS_PROCESSOR) - ItemProcessor reportUnclusteredSSProcessor, + ItemProcessor reportUnclusteredSSProcessor, @Qualifier(PENDING_MERGE_AND_SPLIT_REPORTER) - ItemWriter pendingMergeSplitReporter, + ItemWriter pendingMergeSplitReporter, @Qualifier(PROGRESS_LISTENER) StepExecutionListener progressListener, - StepBuilderFactory stepBuilderFactory, + JobRepository jobRepository, PlatformTransactionManager transactionManager, SimpleCompletionPolicy chunkSizeCompletionPolicy) { - return stepBuilderFactory.get(REPORT_UNCLUSTERED_SS_AND_PENDING_MERGES_AND_SPLITS_STEP) - .chunk(chunkSizeCompletionPolicy) + return new StepBuilder(REPORT_UNCLUSTERED_SS_AND_PENDING_MERGES_AND_SPLITS_STEP, jobRepository) + .chunk(chunkSizeCompletionPolicy, transactionManager) .reader(remappedSSReader) .processor(reportUnclusteredSSProcessor) .writer(pendingMergeSplitReporter) @@ -111,12 +113,12 @@ public Step reportUnclusteredSSAndPendingMergeSplitStep( public Step reportExtraneousRSStep( @Qualifier(REMAPPED_RS_READER) ItemStreamReader remappedRSReader, @Qualifier(EXTRANEOUS_RS_REPORTER) - ItemWriter extraneousRSReporter, + ItemWriter extraneousRSReporter, @Qualifier(PROGRESS_LISTENER) StepExecutionListener progressListener, - StepBuilderFactory stepBuilderFactory, + JobRepository jobRepository, PlatformTransactionManager transactionManager, SimpleCompletionPolicy chunkSizeCompletionPolicy) { - return stepBuilderFactory.get(REPORT_EXTRANEOUS_RS_STEP) - .chunk(chunkSizeCompletionPolicy) + return new StepBuilder(REPORT_EXTRANEOUS_RS_STEP, jobRepository) + .chunk(chunkSizeCompletionPolicy, transactionManager) .reader(remappedRSReader) .writer(extraneousRSReporter) .listener(progressListener) @@ -126,11 +128,11 @@ public Step reportExtraneousRSStep( @Bean(CLUSTERING_QC_JOB) public Job ClusteringQCJob( @Qualifier(REPORT_UNCLUSTERED_SS_AND_PENDING_MERGES_AND_SPLITS_STEP) - Step reportUnclusteredSSAndPendingMergeSplitStep, + Step reportUnclusteredSSAndPendingMergeSplitStep, @Qualifier(REPORT_EXTRANEOUS_RS_STEP) - Step reportExtraneousRSStep, - JobBuilderFactory jobBuilderFactory) { - return jobBuilderFactory.get(CLUSTERING_QC_JOB) + Step reportExtraneousRSStep, + JobRepository jobRepository) { + return new JobBuilder(CLUSTERING_QC_JOB, jobRepository) .incrementer(new RunIdIncrementer()) .start(reportUnclusteredSSAndPendingMergeSplitStep) .next(reportExtraneousRSStep) diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/qc/DuplicateRSAccQCJobConfiguration.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/qc/DuplicateRSAccQCJobConfiguration.java index 1323233dd..48294ee59 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/qc/DuplicateRSAccQCJobConfiguration.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/qc/DuplicateRSAccQCJobConfiguration.java @@ -3,8 +3,9 @@ import org.springframework.batch.core.Job; import org.springframework.batch.core.Step; import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; -import org.springframework.batch.core.configuration.annotation.JobBuilderFactory; +import org.springframework.batch.core.job.builder.JobBuilder; import org.springframework.batch.core.launch.support.RunIdIncrementer; +import org.springframework.batch.core.repository.JobRepository; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; @@ -22,8 +23,8 @@ public class DuplicateRSAccQCJobConfiguration { private Step duplicateRSAccQCStep; @Bean(DUPLICATE_RS_ACC_QC_JOB) - public Job duplicateRSAccQCJob(JobBuilderFactory jobBuilderFactory) { - return jobBuilderFactory.get(DUPLICATE_RS_ACC_QC_JOB) + public Job duplicateRSAccQCJob(JobRepository jobRepository) { + return new JobBuilder(DUPLICATE_RS_ACC_QC_JOB, jobRepository) .incrementer(new RunIdIncrementer()) .start(duplicateRSAccQCStep) .build(); diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/qc/NewClusteredVariantsQCJobConfiguration.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/qc/NewClusteredVariantsQCJobConfiguration.java index 9c3cb7d3e..beddc0eaa 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/qc/NewClusteredVariantsQCJobConfiguration.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/qc/NewClusteredVariantsQCJobConfiguration.java @@ -18,9 +18,10 @@ import org.springframework.batch.core.Job; import org.springframework.batch.core.Step; import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; -import org.springframework.batch.core.configuration.annotation.JobBuilderFactory; -import org.springframework.batch.core.configuration.annotation.StepBuilderFactory; +import org.springframework.batch.core.job.builder.JobBuilder; import org.springframework.batch.core.launch.support.RunIdIncrementer; +import org.springframework.batch.core.repository.JobRepository; +import org.springframework.batch.core.step.builder.StepBuilder; import org.springframework.batch.item.ItemStreamReader; import org.springframework.batch.item.ItemWriter; import org.springframework.batch.item.file.FlatFileItemReader; @@ -31,7 +32,7 @@ import org.springframework.context.annotation.Configuration; import org.springframework.core.io.FileSystemResource; import org.springframework.data.mongodb.core.MongoTemplate; - +import org.springframework.transaction.PlatformTransactionManager; import uk.ac.ebi.eva.accession.clustering.batch.io.qc.MissingCveReporter; import uk.ac.ebi.eva.accession.clustering.batch.io.qc.RSHashPair; @@ -79,20 +80,19 @@ public MissingCveReporter missingCveReporter(MongoTemplate mongoTemplate) { public Step reportMissingCveStep( @Qualifier(RS_REPORT_READER) ItemStreamReader rsReportReader, @Qualifier(MISSING_CVE_REPORTER) ItemWriter missingCveReporter, - StepBuilderFactory stepBuilderFactory, + JobRepository jobRepository, PlatformTransactionManager transactionManager, SimpleCompletionPolicy chunkSizeCompletionPolicy) { - return stepBuilderFactory.get(REPORT_MISSING_CVE_STEP) - .chunk(chunkSizeCompletionPolicy) - .reader(rsReportReader) - .writer(missingCveReporter) - .build(); + return new StepBuilder(REPORT_MISSING_CVE_STEP, jobRepository) + .chunk(chunkSizeCompletionPolicy, transactionManager) + .reader(rsReportReader) + .writer(missingCveReporter) + .build(); } @Bean(NEW_CLUSTERED_VARIANTS_QC_JOB) - public Job ClusteringQCJob( - @Qualifier(REPORT_MISSING_CVE_STEP) Step reportMissingCveStep, - JobBuilderFactory jobBuilderFactory) { - return jobBuilderFactory.get(NEW_CLUSTERED_VARIANTS_QC_JOB) + public Job ClusteringQCJob(@Qualifier(REPORT_MISSING_CVE_STEP) Step reportMissingCveStep, + JobRepository jobRepository) { + return new JobBuilder(NEW_CLUSTERED_VARIANTS_QC_JOB, jobRepository) .incrementer(new RunIdIncrementer()) .start(reportMissingCveStep) .build(); diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/steps/AccessioningShutdownStepConfiguration.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/steps/AccessioningShutdownStepConfiguration.java index 1c67b1b0e..766cf292b 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/steps/AccessioningShutdownStepConfiguration.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/steps/AccessioningShutdownStepConfiguration.java @@ -2,10 +2,12 @@ import org.springframework.batch.core.Step; import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; -import org.springframework.batch.core.configuration.annotation.StepBuilderFactory; +import org.springframework.batch.core.repository.JobRepository; +import org.springframework.batch.core.step.builder.StepBuilder; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.transaction.PlatformTransactionManager; import uk.ac.ebi.eva.accession.core.service.nonhuman.ClusteredVariantAccessioningService; import uk.ac.ebi.eva.accession.core.service.nonhuman.SubmittedVariantAccessioningService; @@ -21,13 +23,13 @@ public class AccessioningShutdownStepConfiguration { private ClusteredVariantAccessioningService clusteredVariantAccessioningService; @Bean(ACCESSIONING_SHUTDOWN_STEP) - public Step accessioningShutDownStep(StepBuilderFactory stepBuilderFactory) { - return stepBuilderFactory.get(ACCESSIONING_SHUTDOWN_STEP) + public Step accessioningShutDownStep(JobRepository jobRepository, PlatformTransactionManager transactionManager) { + return new StepBuilder(ACCESSIONING_SHUTDOWN_STEP, jobRepository) .tasklet((contribution, chunkContext) -> { submittedVariantAccessioningService.shutDownAccessionGenerator(); clusteredVariantAccessioningService.shutDownAccessionGenerator(); return null; - }) + }, transactionManager) .build(); } } \ No newline at end of file diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/steps/ClusteringFromMongoStepConfiguration.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/steps/ClusteringFromMongoStepConfiguration.java index 3e1701265..41417be99 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/steps/ClusteringFromMongoStepConfiguration.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/steps/ClusteringFromMongoStepConfiguration.java @@ -19,7 +19,8 @@ import org.springframework.batch.core.Step; import org.springframework.batch.core.StepExecutionListener; import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; -import org.springframework.batch.core.configuration.annotation.StepBuilderFactory; +import org.springframework.batch.core.repository.JobRepository; +import org.springframework.batch.core.step.builder.StepBuilder; import org.springframework.batch.core.step.tasklet.TaskletStep; import org.springframework.batch.item.ItemReader; import org.springframework.batch.item.ItemStreamReader; @@ -28,38 +29,38 @@ import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; - +import org.springframework.transaction.PlatformTransactionManager; import uk.ac.ebi.eva.accession.clustering.batch.io.ListOfListItemWriter; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantOperationEntity; import java.util.List; -import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.CLUSTERED_CLUSTERING_WRITER_JOB_EXECUTION_SETTER; -import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.NON_CLUSTERED_CLUSTERING_WRITER_JOB_EXECUTION_SETTER; -import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.RS_SPLIT_WRITER_JOB_EXECUTION_SETTER; -import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.STUDY_CLUSTERING_MONGO_READER; -import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.STUDY_CLUSTERING_STEP; -import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.TARGET_SS_READER_FOR_NEW_BACKPROP_RS; -import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.TARGET_SS_READER_FOR_SPLIT_OR_MERGED_BACKPROP_RS; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.BACK_PROPAGATED_RS_WRITER; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.BACK_PROPAGATE_NEW_RS_STEP; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.BACK_PROPAGATE_SPLIT_OR_MERGED_RS_STEP; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.CLEAR_RS_MERGE_AND_SPLIT_CANDIDATES; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.CLEAR_RS_MERGE_AND_SPLIT_CANDIDATES_STEP; +import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.CLUSTERED_CLUSTERING_WRITER; +import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.CLUSTERED_CLUSTERING_WRITER_JOB_EXECUTION_SETTER; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.CLUSTERED_VARIANTS_MONGO_READER; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.CLUSTERING_CLUSTERED_VARIANTS_FROM_MONGO_STEP; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.CLUSTERING_NON_CLUSTERED_VARIANTS_FROM_MONGO_STEP; -import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.CLUSTERED_CLUSTERING_WRITER; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.NON_CLUSTERED_CLUSTERING_WRITER; +import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.NON_CLUSTERED_CLUSTERING_WRITER_JOB_EXECUTION_SETTER; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.NON_CLUSTERED_VARIANTS_MONGO_READER; -import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.PROCESS_RS_SPLIT_CANDIDATES_STEP; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.PROCESS_RS_MERGE_CANDIDATES_STEP; +import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.PROCESS_RS_SPLIT_CANDIDATES_STEP; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.PROGRESS_LISTENER; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.RS_MERGE_CANDIDATES_READER; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.RS_MERGE_WRITER; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.RS_SPLIT_CANDIDATES_READER; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.RS_SPLIT_WRITER; +import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.RS_SPLIT_WRITER_JOB_EXECUTION_SETTER; +import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.STUDY_CLUSTERING_MONGO_READER; +import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.STUDY_CLUSTERING_STEP; +import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.TARGET_SS_READER_FOR_NEW_BACKPROP_RS; +import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.TARGET_SS_READER_FOR_SPLIT_OR_MERGED_BACKPROP_RS; @Configuration @EnableBatchProcessing @@ -71,10 +72,10 @@ public Step clusteringClusteredVariantStepMongoReader( @Qualifier(CLUSTERED_CLUSTERING_WRITER) ItemWriter submittedVariantWriter, @Qualifier(PROGRESS_LISTENER) StepExecutionListener progressListener, @Qualifier(CLUSTERED_CLUSTERING_WRITER_JOB_EXECUTION_SETTER) StepExecutionListener clusteredClusteringWriterJobExecutionSetter, - StepBuilderFactory stepBuilderFactory, + JobRepository jobRepository, PlatformTransactionManager transactionManager, SimpleCompletionPolicy chunkSizeCompletionPolicy) { - TaskletStep step = stepBuilderFactory.get(CLUSTERING_CLUSTERED_VARIANTS_FROM_MONGO_STEP) - .chunk(chunkSizeCompletionPolicy) + TaskletStep step = new StepBuilder(CLUSTERING_CLUSTERED_VARIANTS_FROM_MONGO_STEP, jobRepository) + .chunk(chunkSizeCompletionPolicy, transactionManager) .reader(mongoReader) .writer(submittedVariantWriter) .listener(progressListener) @@ -86,51 +87,49 @@ public Step clusteringClusteredVariantStepMongoReader( @Bean(PROCESS_RS_MERGE_CANDIDATES_STEP) public Step processRSMergeCandidatesStep( @Qualifier(RS_MERGE_CANDIDATES_READER) - ItemReader rsMergeCandidatesReader, + ItemReader rsMergeCandidatesReader, @Qualifier(RS_MERGE_WRITER) ItemWriter rsMergeWriter, @Qualifier(PROGRESS_LISTENER) StepExecutionListener progressListener, - StepBuilderFactory stepBuilderFactory, + JobRepository jobRepository, PlatformTransactionManager transactionManager, SimpleCompletionPolicy chunkSizeCompletionPolicy) { - TaskletStep step = stepBuilderFactory.get(PROCESS_RS_MERGE_CANDIDATES_STEP) - .chunk( - chunkSizeCompletionPolicy) - .reader(rsMergeCandidatesReader) - .writer(rsMergeWriter) - .listener(progressListener) - .build(); + TaskletStep step = new StepBuilder(PROCESS_RS_MERGE_CANDIDATES_STEP, jobRepository) + .chunk(chunkSizeCompletionPolicy, transactionManager) + .reader(rsMergeCandidatesReader) + .writer(rsMergeWriter) + .listener(progressListener) + .build(); return step; } @Bean(PROCESS_RS_SPLIT_CANDIDATES_STEP) public Step processRSSplitCandidatesStep( @Qualifier(RS_SPLIT_CANDIDATES_READER) - ItemReader rsSplitCandidatesReader, + ItemReader rsSplitCandidatesReader, @Qualifier(RS_SPLIT_WRITER) ItemWriter rsSplitWriter, @Qualifier(PROGRESS_LISTENER) StepExecutionListener progressListener, @Qualifier(RS_SPLIT_WRITER_JOB_EXECUTION_SETTER) StepExecutionListener rsSplitWriterJobExecutionSetter, - StepBuilderFactory stepBuilderFactory, + JobRepository jobRepository, PlatformTransactionManager transactionManager, SimpleCompletionPolicy chunkSizeCompletionPolicy) { - TaskletStep step = stepBuilderFactory.get(PROCESS_RS_SPLIT_CANDIDATES_STEP) - .chunk( - chunkSizeCompletionPolicy) - .reader(rsSplitCandidatesReader) - .writer(rsSplitWriter) - .listener(progressListener) - .listener(rsSplitWriterJobExecutionSetter) - .build(); + TaskletStep step = new StepBuilder(PROCESS_RS_SPLIT_CANDIDATES_STEP, jobRepository) + .chunk(chunkSizeCompletionPolicy, transactionManager) + .reader(rsSplitCandidatesReader) + .writer(rsSplitWriter) + .listener(progressListener) + .listener(rsSplitWriterJobExecutionSetter) + .build(); return step; } @Bean(CLEAR_RS_MERGE_AND_SPLIT_CANDIDATES_STEP) public Step clearRSMergeAndSplitCandidatesStep( @Qualifier(CLEAR_RS_MERGE_AND_SPLIT_CANDIDATES) ItemWriter clearRSMergeAndSplitCandidates, - StepBuilderFactory stepBuilderFactory, + JobRepository jobRepository, PlatformTransactionManager transactionManager, SimpleCompletionPolicy chunkSizeCompletionPolicy) { - TaskletStep step = stepBuilderFactory.get(CLEAR_RS_MERGE_AND_SPLIT_CANDIDATES_STEP) - .chunk(chunkSizeCompletionPolicy) - .reader(new SingleItemReader()) - .writer(clearRSMergeAndSplitCandidates) - .build(); + TaskletStep step = new StepBuilder(CLEAR_RS_MERGE_AND_SPLIT_CANDIDATES_STEP, jobRepository) + .chunk(chunkSizeCompletionPolicy, transactionManager) + .reader(new SingleItemReader()) + .writer(clearRSMergeAndSplitCandidates) + .build(); return step; } @@ -139,6 +138,7 @@ public Step clearRSMergeAndSplitCandidatesStep( // so that the writer to clear RS merge and split candidate entries can proceed public static class SingleItemReader implements ItemReader { static boolean firstTime = true; + @Override public Object read() throws Exception { if (firstTime) { @@ -155,10 +155,10 @@ public Step clusteringNonClusteredVariantStepMongoReader( @Qualifier(NON_CLUSTERED_CLUSTERING_WRITER) ItemWriter submittedVariantWriter, @Qualifier(PROGRESS_LISTENER) StepExecutionListener progressListener, @Qualifier(NON_CLUSTERED_CLUSTERING_WRITER_JOB_EXECUTION_SETTER) StepExecutionListener nonClusteredClusteringWriterJobExecutionSetter, - StepBuilderFactory stepBuilderFactory, + JobRepository jobRepository, PlatformTransactionManager transactionManager, SimpleCompletionPolicy chunkSizeCompletionPolicy) { - TaskletStep step = stepBuilderFactory.get(CLUSTERING_NON_CLUSTERED_VARIANTS_FROM_MONGO_STEP) - .chunk(chunkSizeCompletionPolicy) + TaskletStep step = new StepBuilder(CLUSTERING_NON_CLUSTERED_VARIANTS_FROM_MONGO_STEP, jobRepository) + .chunk(chunkSizeCompletionPolicy, transactionManager) .reader(mongoReader) .writer(submittedVariantWriter) .listener(progressListener) @@ -170,34 +170,32 @@ public Step clusteringNonClusteredVariantStepMongoReader( @Bean(BACK_PROPAGATE_NEW_RS_STEP) public Step backPropagateNewRSStep( @Qualifier(TARGET_SS_READER_FOR_NEW_BACKPROP_RS) - ItemStreamReader backPropagatedNewRSTargetReader, + ItemStreamReader backPropagatedNewRSTargetReader, @Qualifier(BACK_PROPAGATED_RS_WRITER) ItemWriter backPropagatedRSWriter, @Qualifier(PROGRESS_LISTENER) StepExecutionListener progressListener, - StepBuilderFactory stepBuilderFactory, + JobRepository jobRepository, PlatformTransactionManager transactionManager, SimpleCompletionPolicy chunkSizeCompletionPolicy) { - TaskletStep step = stepBuilderFactory.get(BACK_PROPAGATE_NEW_RS_STEP) - .chunk( - chunkSizeCompletionPolicy) - .reader(backPropagatedNewRSTargetReader) - .writer(backPropagatedRSWriter) - .listener(progressListener) - .build(); + TaskletStep step = new StepBuilder(BACK_PROPAGATE_NEW_RS_STEP, jobRepository) + .chunk(chunkSizeCompletionPolicy, transactionManager) + .reader(backPropagatedNewRSTargetReader) + .writer(backPropagatedRSWriter) + .listener(progressListener) + .build(); return step; } @Bean(BACK_PROPAGATE_SPLIT_OR_MERGED_RS_STEP) public Step backPropagateSplitAndMergedRSStep( @Qualifier(TARGET_SS_READER_FOR_SPLIT_OR_MERGED_BACKPROP_RS) - ItemStreamReader> splitOrMergedRSReader, + ItemStreamReader> splitOrMergedRSReader, @Qualifier(BACK_PROPAGATED_RS_WRITER) ItemWriter backPropagatedRSWriter, @Qualifier(PROGRESS_LISTENER) StepExecutionListener progressListener, - StepBuilderFactory stepBuilderFactory, + JobRepository jobRepository, PlatformTransactionManager transactionManager, SimpleCompletionPolicy chunkSizeCompletionPolicy) { - TaskletStep step = stepBuilderFactory.get(BACK_PROPAGATE_SPLIT_OR_MERGED_RS_STEP) - ., List>chunk( - chunkSizeCompletionPolicy) + TaskletStep step = new StepBuilder(BACK_PROPAGATE_SPLIT_OR_MERGED_RS_STEP, jobRepository) + ., List>chunk(chunkSizeCompletionPolicy, transactionManager) .reader(splitOrMergedRSReader) - // Spring needs this wrapping to flatten the List> from the processor + // Spring needs this wrapping to flatten the List> from the processor .writer(new ListOfListItemWriter<>(backPropagatedRSWriter)) .listener(progressListener) .build(); @@ -210,15 +208,15 @@ public Step studyClusteringStep( @Qualifier(NON_CLUSTERED_CLUSTERING_WRITER) ItemWriter submittedVariantWriter, @Qualifier(PROGRESS_LISTENER) StepExecutionListener progressListener, @Qualifier(NON_CLUSTERED_CLUSTERING_WRITER_JOB_EXECUTION_SETTER) StepExecutionListener nonClusteredClusteringWriterJobExecutionSetter, - StepBuilderFactory stepBuilderFactory, + JobRepository jobRepository, PlatformTransactionManager transactionManager, SimpleCompletionPolicy chunkSizeCompletionPolicy) { - TaskletStep step = stepBuilderFactory.get(STUDY_CLUSTERING_STEP) - .chunk(chunkSizeCompletionPolicy) - .reader(mongoReader) - .writer(submittedVariantWriter) - .listener(progressListener) - .listener(nonClusteredClusteringWriterJobExecutionSetter) - .build(); + TaskletStep step = new StepBuilder(STUDY_CLUSTERING_STEP, jobRepository) + .chunk(chunkSizeCompletionPolicy, transactionManager) + .reader(mongoReader) + .writer(submittedVariantWriter) + .listener(progressListener) + .listener(nonClusteredClusteringWriterJobExecutionSetter) + .build(); return step; } } diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/steps/RSAccessionRecoveryStepConfiguration.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/steps/RSAccessionRecoveryStepConfiguration.java index a0514439d..f484eda83 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/steps/RSAccessionRecoveryStepConfiguration.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/steps/RSAccessionRecoveryStepConfiguration.java @@ -2,11 +2,13 @@ import org.springframework.batch.core.Step; import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; -import org.springframework.batch.core.configuration.annotation.StepBuilderFactory; +import org.springframework.batch.core.repository.JobRepository; +import org.springframework.batch.core.step.builder.StepBuilder; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.transaction.PlatformTransactionManager; import uk.ac.ebi.eva.accession.clustering.batch.recovery.RSAccessionRecoveryService; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.RS_ACCESSION_RECOVERY_SERVICE; @@ -20,12 +22,13 @@ public class RSAccessionRecoveryStepConfiguration { private RSAccessionRecoveryService RSAccessionRecoveryService; @Bean(RS_ACCESSION_RECOVERY_STEP) - public Step monotonicAccessionRecoveryAgentCategoryRSStep(StepBuilderFactory stepBuilderFactory) { - return stepBuilderFactory.get(RS_ACCESSION_RECOVERY_STEP) + public Step monotonicAccessionRecoveryAgentCategoryRSStep(JobRepository jobRepository, + PlatformTransactionManager transactionManager) { + return new StepBuilder(RS_ACCESSION_RECOVERY_STEP, jobRepository) .tasklet((contribution, chunkContext) -> { RSAccessionRecoveryService.runRecoveryForCategoryRS(); return null; - }) + }, transactionManager) .build(); } } diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/steps/qc/DuplicateRSAccQCStepConfiguration.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/steps/qc/DuplicateRSAccQCStepConfiguration.java index 9af162201..2c0afce12 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/steps/qc/DuplicateRSAccQCStepConfiguration.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/steps/qc/DuplicateRSAccQCStepConfiguration.java @@ -2,7 +2,8 @@ import org.springframework.batch.core.Step; import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; -import org.springframework.batch.core.configuration.annotation.StepBuilderFactory; +import org.springframework.batch.core.repository.JobRepository; +import org.springframework.batch.core.step.builder.StepBuilder; import org.springframework.batch.core.step.tasklet.TaskletStep; import org.springframework.batch.item.ItemProcessor; import org.springframework.batch.item.ItemStreamReader; @@ -11,6 +12,7 @@ import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.transaction.PlatformTransactionManager; import uk.ac.ebi.eva.accession.clustering.batch.io.qc.DuplicateRSAccQCResult; import java.util.List; @@ -37,11 +39,11 @@ public class DuplicateRSAccQCStepConfiguration { private ItemWriter> duplicateRSAccQCWriter; @Bean(DUPLICATE_RS_ACC_QC_STEP) - public Step duplicateRSAccQCStep(StepBuilderFactory stepBuilderFactory) { - TaskletStep step = stepBuilderFactory.get(DUPLICATE_RS_ACC_QC_STEP) + public Step duplicateRSAccQCStep(JobRepository jobRepository, PlatformTransactionManager transactionManager) { + TaskletStep step = new StepBuilder(DUPLICATE_RS_ACC_QC_STEP, jobRepository) // hardcoded the chunk size as 1, as the reader takes care of accumulating // and sending the chunk size (defined in properties file) elements to the processor - ., List>chunk(1) + ., List>chunk(1, transactionManager) .reader(duplicateRSAccFileReader) .processor(duplicateRSAccQCProcessor) .writer(duplicateRSAccQCWriter) diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/metric/ClusteringMetricCompute.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/metric/ClusteringMetricCompute.java index 1491f700f..d32187071 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/metric/ClusteringMetricCompute.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/metric/ClusteringMetricCompute.java @@ -1,7 +1,8 @@ package uk.ac.ebi.eva.accession.clustering.metric; -import org.codehaus.jettison.json.JSONException; -import org.codehaus.jettison.json.JSONObject; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; import org.springframework.web.client.RestTemplate; import uk.ac.ebi.eva.metrics.count.CountServiceParameters; import uk.ac.ebi.eva.metrics.metric.BaseMetricCompute; @@ -11,6 +12,8 @@ import java.util.stream.Collectors; public class ClusteringMetricCompute extends BaseMetricCompute { + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + private static final String PROCESS = "clustering"; private final String assembly; @@ -33,12 +36,12 @@ public List getMetrics() { public String getIdentifier() { try { - JSONObject identifier = new JSONObject(); + ObjectNode identifier = OBJECT_MAPPER.createObjectNode(); identifier.put("assembly", assembly); - identifier.put("projects", projects); - return identifier.toString(); - } catch (JSONException jsonException) { - throw new RuntimeException("Could not create Identifier for Clustering Counts. Error ", jsonException); + identifier.putPOJO("projects", projects); + return OBJECT_MAPPER.writeValueAsString(identifier); + } catch (JsonProcessingException e) { + throw new RuntimeException("Could not create Identifier for Clustering Counts. Error ", e); } } diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/parameters/InputParameters.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/parameters/InputParameters.java index a2358de0b..c2128afb8 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/parameters/InputParameters.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/parameters/InputParameters.java @@ -15,7 +15,6 @@ */ package uk.ac.ebi.eva.accession.clustering.parameters; -import org.apache.commons.collections.CollectionUtils; import org.springframework.batch.core.JobParameters; import org.springframework.batch.core.JobParametersBuilder; import org.springframework.batch.core.JobParametersInvalidException; @@ -95,7 +94,7 @@ public JobParameters toJobParameters() throws JobParametersInvalidException { if (projects.stream().anyMatch(s -> s.contains(","))) { throw new JobParametersInvalidException("Can't have commas in project accessions"); } - String projectsString = CollectionUtils.isEmpty(projects) ? "" : String.join(",", projects); + String projectsString = (projects == null || projects.isEmpty()) ? "" : String.join(",", projects); if (projectsString.length() > 250) { throw new JobParametersInvalidException("Max length of projects parameter is 250 characters"); } diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/runner/ClusteringCommandLineRunner.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/runner/ClusteringCommandLineRunner.java index 8b089f129..25e60ba7f 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/runner/ClusteringCommandLineRunner.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/runner/ClusteringCommandLineRunner.java @@ -25,7 +25,6 @@ import org.springframework.batch.core.JobParametersInvalidException; import org.springframework.batch.core.explore.JobExplorer; import org.springframework.batch.core.launch.JobLauncher; -import org.springframework.batch.core.launch.JobParametersNotFoundException; import org.springframework.batch.core.repository.JobExecutionAlreadyRunningException; import org.springframework.batch.core.repository.JobInstanceAlreadyCompleteException; import org.springframework.batch.core.repository.JobRepository; @@ -33,7 +32,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.ExitCodeGenerator; -import org.springframework.boot.autoconfigure.batch.JobLauncherCommandLineRunner; +import org.springframework.boot.autoconfigure.batch.JobLauncherApplicationRunner; import org.springframework.context.ApplicationEventPublisherAware; import org.springframework.stereotype.Component; import org.springframework.util.PatternMatchUtils; @@ -49,7 +48,7 @@ import java.util.Collection; @Component -public class ClusteringCommandLineRunner extends JobLauncherCommandLineRunner implements +public class ClusteringCommandLineRunner extends JobLauncherApplicationRunner implements ApplicationEventPublisherAware, ExitCodeGenerator { private static final Logger logger = LoggerFactory.getLogger(ClusteringCommandLineRunner.class); @@ -90,9 +89,9 @@ public void setJobs(Collection jobs) { } @Override - public void setJobNames(String jobName) { + public void setJobName(String jobName) { this.jobName = jobName; - super.setJobNames(jobName); + super.setJobName(jobName); } @Override @@ -116,16 +115,15 @@ public void run(String... args) throws JobExecutionException { JobStatusManager.checkIfPropertiesHaveBeenProvided(jobParameters); if (inputParameters.isForceRestart()) { JobExecution previousJobExecution = CommandLineRunnerUtils.getLastJobExecution(jobName, jobExplorer, - jobParameters); + jobParameters); CommandLineRunnerUtils.markPreviousJobAsFailed(jobName, jobRepository, previousJobExecution != null ? previousJobExecution.getJobParameters() : jobParameters); - } - else { + } else { jobParameters = CommandLineRunnerUtils.addRunIDToJobParameters(jobName, jobExplorer, jobParameters); } launchJob(jobParameters); } catch (NoJobToExecuteException | NoParametersHaveBeenPassedException | NoPreviousJobExecutionException - | UnknownJobException | JobParametersInvalidException | JobExecutionAlreadyRunningException e) { + | UnknownJobException | JobParametersInvalidException | JobExecutionAlreadyRunningException e) { logger.error(e.getMessage()); logger.debug("Error trace", e); abnormalExit = true; @@ -147,8 +145,7 @@ private void launchJob(JobParameters jobParameters) throws JobExecutionException @Override protected void execute(Job job, JobParameters jobParameters) throws JobExecutionAlreadyRunningException, - JobRestartException, JobInstanceAlreadyCompleteException, JobParametersInvalidException, - JobParametersNotFoundException { + JobRestartException, JobInstanceAlreadyCompleteException, JobParametersInvalidException { logger.info("Running job '" + jobName + "' with parameters: " + jobParameters); super.execute(job, jobParameters); } diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/BackPropagatedRSWriterTest.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/BackPropagatedRSWriterTest.java index f4df28eaf..a255bf923 100644 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/BackPropagatedRSWriterTest.java +++ b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/BackPropagatedRSWriterTest.java @@ -15,49 +15,53 @@ */ package uk.ac.ebi.eva.accession.clustering.batch.io; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; -import org.junit.*; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.batch.item.Chunk; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.context.ApplicationContext; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.ampt2d.commons.accession.core.exceptions.AccessionCouldNotBeGeneratedException; import uk.ac.ebi.ampt2d.commons.accession.core.models.AccessionWrapper; import uk.ac.ebi.ampt2d.commons.accession.hashing.SHA1HashingFunction; import uk.ac.ebi.eva.accession.clustering.test.configuration.BatchTestConfiguration; -import uk.ac.ebi.eva.accession.clustering.test.rule.FixSpringMongoDbRule; +import uk.ac.ebi.eva.accession.clustering.test.configuration.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.model.ClusteredVariant; import uk.ac.ebi.eva.accession.core.model.IClusteredVariant; import uk.ac.ebi.eva.accession.core.model.ISubmittedVariant; import uk.ac.ebi.eva.accession.core.model.SubmittedVariant; -import uk.ac.ebi.eva.accession.core.model.dbsnp.*; -import uk.ac.ebi.eva.accession.core.model.eva.*; +import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpClusteredVariantEntity; +import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpSubmittedVariantEntity; +import uk.ac.ebi.eva.accession.core.model.eva.ClusteredVariantEntity; +import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; import uk.ac.ebi.eva.accession.core.service.nonhuman.SubmittedVariantAccessioningService; import uk.ac.ebi.eva.accession.core.summary.ClusteredVariantSummaryFunction; import uk.ac.ebi.eva.accession.core.summary.SubmittedVariantSummaryFunction; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.commons.core.models.VariantType; import uk.ac.ebi.eva.metrics.metric.MetricCompute; -import java.util.*; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Objects; import java.util.function.Function; import java.util.stream.Collectors; -import static org.junit.Assert.*; -import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.CLUSTERED_CLUSTERING_WRITER; -@RunWith(SpringRunner.class) -@ContextConfiguration(classes={BatchTestConfiguration.class}) +@ExtendWith(SpringExtension.class) +@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class,}) @TestPropertySource("classpath:backpropagation-test.properties") -public class BackPropagatedRSWriterTest { - - private static final String TEST_DB = "test-db"; - +public class BackPropagatedRSWriterTest extends MongoTestContainerHelper { private static final int TAXONOMY = 60711; private static final String ASM1 = "asm1"; @@ -76,10 +80,6 @@ public class BackPropagatedRSWriterTest { @Autowired private MongoTemplate mongoTemplate; - //Required by nosql-unit - @Autowired - private ApplicationContext applicationContext; - @Autowired private SubmittedVariantAccessioningService submittedVariantAccessioningService; @@ -87,10 +87,6 @@ public class BackPropagatedRSWriterTest { @Qualifier(CLUSTERED_CLUSTERING_WRITER) private ClusteringWriter clusteringWriter; - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName(TEST_DB).build()); - private static class RSLocus { String assembly; String contig; @@ -113,12 +109,12 @@ public String getHash() { } } - @Before + @BeforeEach public void setUp() throws Exception { mongoTemplate.getDb().drop(); } - @After + @AfterEach public void tearDown() { mongoTemplate.getDb().drop(); } @@ -146,7 +142,7 @@ public void runBackPropagationAcrossMultipleRemappings() throws AccessionCouldNo BackPropagatedRSWriter backPropagatedRSWriter = new BackPropagatedRSWriter(ASM2, this.clusteringWriter, this.submittedVariantAccessioningService, this.mongoTemplate, this.metricCompute); - backPropagatedRSWriter.write(Arrays.asList(ss2)); + backPropagatedRSWriter.write(Chunk.of(ss2)); // As of T2, see https://docs.google.com/spreadsheets/d/1KQLVCUy-vqXKgkCDt2czX6kuMfsjfCc9uBsS19MZ6dY/edit#rangeid=688023832 assertSSBackPropRSAssociation(ss2.getAccession(), rs2.getAccession(), ASM1); @@ -156,7 +152,7 @@ public void runBackPropagationAcrossMultipleRemappings() throws AccessionCouldNo this.mongoTemplate.insert(Arrays.asList(ss1_asm3_remap, ss2_asm3_remap), DbsnpSubmittedVariantEntity.class); backPropagatedRSWriter = new BackPropagatedRSWriter(ASM3, this.clusteringWriter, this.submittedVariantAccessioningService, this.mongoTemplate, this.metricCompute); - backPropagatedRSWriter.write(Arrays.asList(ss2)); + backPropagatedRSWriter.write(Chunk.of(ss2)); // As of T3, see https://docs.google.com/spreadsheets/d/1KQLVCUy-vqXKgkCDt2czX6kuMfsjfCc9uBsS19MZ6dY/edit#rangeid=490224900 assertSSBackPropRSAssociation(ss2.getAccession(), rs1.getAccession(), ASM1); @@ -169,7 +165,7 @@ public void runBackPropagationAcrossMultipleRemappings() throws AccessionCouldNo this.mongoTemplate.insert(Arrays.asList(ss1_asm4_remap, ss2_asm4_remap), DbsnpSubmittedVariantEntity.class); backPropagatedRSWriter = new BackPropagatedRSWriter(ASM4, this.clusteringWriter, this.submittedVariantAccessioningService, this.mongoTemplate, this.metricCompute); - backPropagatedRSWriter.write(Arrays.asList(ss2)); + backPropagatedRSWriter.write(Chunk.of(ss2)); // As of T4, see https://docs.google.com/spreadsheets/d/1KQLVCUy-vqXKgkCDt2czX6kuMfsjfCc9uBsS19MZ6dY/edit#rangeid=48828480 assertSSBackPropRSAssociation(ss2.getAccession(), rs3.getAccession(), ASM1); } @@ -190,7 +186,7 @@ private void assertSSBackPropRSAssociation(Long ssID, Long expectedBackPropRS, S private SubmittedVariantEntity createSS(Long ssAccession, Long rsAccession, RSLocus rsLocus, String reference, String alternate, boolean remappedFromAnotherAssembly) { - Function hashingFunction = new SubmittedVariantSummaryFunction().andThen( + Function hashingFunction = new SubmittedVariantSummaryFunction().andThen( new SHA1HashingFunction()); SubmittedVariant submittedVariant = new SubmittedVariant(rsLocus.assembly, TAXONOMY, PROJECT, rsLocus.contig, rsLocus.start, reference, alternate, rsAccession); @@ -204,7 +200,7 @@ private SubmittedVariantEntity createSS(Long ssAccession, Long rsAccession, RSLo } private ClusteredVariantEntity createRS(Long rsAccession, RSLocus rsLocus) { - Function hashingFunction = new ClusteredVariantSummaryFunction().andThen( + Function hashingFunction = new ClusteredVariantSummaryFunction().andThen( new SHA1HashingFunction()); ClusteredVariant clusteredVariant = new ClusteredVariant(rsLocus.assembly, TAXONOMY, rsLocus.contig, rsLocus.start, rsLocus.type, false, null); diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/ClusteredVariantSplittingPolicyTest.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/ClusteredVariantSplittingPolicyTest.java index ceb784278..f8a4ebc98 100644 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/ClusteredVariantSplittingPolicyTest.java +++ b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/ClusteredVariantSplittingPolicyTest.java @@ -15,9 +15,9 @@ */ package uk.ac.ebi.eva.accession.clustering.batch.io; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.springframework.test.context.junit4.SpringRunner; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.ampt2d.commons.accession.hashing.SHA1HashingFunction; import uk.ac.ebi.eva.accession.clustering.batch.io.ClusteredVariantSplittingPolicy.SplitDeterminants; @@ -29,9 +29,9 @@ import java.util.function.Function; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; -@RunWith(SpringRunner.class) +@ExtendWith(SpringExtension.class) public class ClusteredVariantSplittingPolicyTest { @Test diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/ClusteringMongoReaderRetryTest.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/ClusteringMongoReaderRetryTest.java index 2ec255230..f3c8e199e 100644 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/ClusteringMongoReaderRetryTest.java +++ b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/ClusteringMongoReaderRetryTest.java @@ -21,16 +21,16 @@ import com.mongodb.client.MongoCursor; import org.bson.Document; import org.bson.conversions.Bson; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Answers; import org.springframework.batch.item.ExecutionContext; import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.data.mongodb.core.convert.MongoConverter; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; @@ -39,8 +39,8 @@ import java.util.List; import java.util.stream.Collectors; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThrows; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; @@ -50,7 +50,7 @@ import static org.mockito.Mockito.when; import static uk.ac.ebi.eva.accession.clustering.batch.io.ClusteringMongoReaderRetryable.MAX_RETRIES; -@RunWith(SpringRunner.class) +@ExtendWith(SpringExtension.class) public class ClusteringMongoReaderRetryTest { private static final String ASSEMBLY = "GCA_000000001.1"; @@ -62,12 +62,12 @@ public class ClusteringMongoReaderRetryTest { @MockBean(answer = Answers.RETURNS_DEEP_STUBS) private MongoTemplate mongoTemplate; - @Before + @BeforeEach public void setUp(){ nonClusteredVariantReader = spy(new ClusteringMongoReaderRetryable(mongoTemplate, ASSEMBLY, CHUNK_SIZE, false)); } - @After + @AfterEach public void tearDown() { nonClusteredVariantReader.close(); } diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/ClusteringMongoReaderTest.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/ClusteringMongoReaderTest.java index 4b7dd33df..c5971393b 100644 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/ClusteringMongoReaderTest.java +++ b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/ClusteringMongoReaderTest.java @@ -15,42 +15,34 @@ */ package uk.ac.ebi.eva.accession.clustering.batch.io; -import com.lordofthejars.nosqlunit.annotation.UsingDataSet; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; -import com.mongodb.MongoClient; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.batch.item.ExecutionContext; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.ApplicationContext; +import org.springframework.core.io.ResourceLoader; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.eva.accession.clustering.test.configuration.MongoTestConfiguration; -import uk.ac.ebi.eva.accession.clustering.test.rule.FixSpringMongoDbRule; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.MongoConfiguration; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; +import uk.ac.ebi.eva.accession.core.utils.MongoTestDataLoader; import java.util.ArrayList; import java.util.List; import java.util.Objects; import static junit.framework.TestCase.assertTrue; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; -@RunWith(SpringRunner.class) +@ExtendWith(SpringExtension.class) @TestPropertySource("classpath:clustering-pipeline-test.properties") -@UsingDataSet(locations = {"/test-data/submittedVariantEntityMongoReader.json", - "/test-data/dbsnpSubmittedVariantEntityMongoReader.json"}) @ContextConfiguration(classes = {MongoConfiguration.class, MongoTestConfiguration.class}) -public class ClusteringMongoReaderTest { - - private static final String TEST_DB = "test-db"; +public class ClusteringMongoReaderTest extends MongoTestContainerHelper { private static final String ASSEMBLY = "GCA_000000001.1"; @@ -72,22 +64,20 @@ public class ClusteringMongoReaderTest { private ClusteringMongoReader nonClusteredVariantReader; - @Autowired - private MongoClient mongoClient; - @Autowired private MongoTemplate mongoTemplate; - //Required by nosql-unit @Autowired - private ApplicationContext applicationContext; + private ResourceLoader resourceLoader; + + @BeforeEach + public void setUp() { + mongoTemplate.getDb().drop(); - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName(TEST_DB).build()); + MongoTestDataLoader mongoTestDataLoader = new MongoTestDataLoader(mongoTemplate, resourceLoader); + mongoTestDataLoader.load("/test-data/submittedVariantEntityMongoReader.json"); + mongoTestDataLoader.load("/test-data/dbsnpSubmittedVariantEntityMongoReader.json"); - @Before - public void setUp(){ ExecutionContext executionContext = new ExecutionContext(); clusteredVariantReader = new ClusteringMongoReader(mongoTemplate, ASSEMBLY, CHUNK_SIZE, true); nonClusteredVariantReader = new ClusteringMongoReader(mongoTemplate, ASSEMBLY, CHUNK_SIZE, false); @@ -95,11 +85,11 @@ public void setUp(){ nonClusteredVariantReader.open(executionContext); } - @After + @AfterEach public void tearDown() { clusteredVariantReader.close(); nonClusteredVariantReader.close(); - mongoClient.dropDatabase(TEST_DB); + mongoTemplate.getDb().drop(); } @Test @@ -122,7 +112,7 @@ public void readAllSubmittedVariantsWithoutRS() { assertEquals(5, variants.size()); assertTrue(variants.stream().anyMatch(x -> Objects.equals(x.getId(), NOT_CLUSTERED_SUBMITTED_VARIANT_ID_EVA))); assertTrue(variants.stream().noneMatch(x -> Objects.equals(x.getId(), - NOT_CLUSTERED_SUBMITTED_VARIANT_ID_DBSNP))); + NOT_CLUSTERED_SUBMITTED_VARIANT_ID_DBSNP))); } private List readIntoList(ClusteringMongoReader reader) { diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/RSMergeAndSplitCandidatesReaderTest.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/RSMergeAndSplitCandidatesReaderTest.java index 8bbf4324a..6d14878c2 100644 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/RSMergeAndSplitCandidatesReaderTest.java +++ b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/RSMergeAndSplitCandidatesReaderTest.java @@ -15,45 +15,39 @@ */ package uk.ac.ebi.eva.accession.clustering.batch.io; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; -import com.mongodb.MongoClient; - -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; - +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.batch.item.ExecutionContext; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.context.ApplicationContext; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; - +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.eva.accession.clustering.configuration.batch.io.RSMergeAndSplitCandidatesReaderConfiguration; import uk.ac.ebi.eva.accession.clustering.test.configuration.BatchTestConfiguration; import uk.ac.ebi.eva.accession.clustering.test.configuration.MongoTestConfiguration; -import uk.ac.ebi.eva.accession.clustering.test.rule.FixSpringMongoDbRule; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantInactiveEntity; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantOperationEntity; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.commons.mongodb.readers.MongoDbCursorItemReader; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.RS_MERGE_CANDIDATES_READER; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.RS_SPLIT_CANDIDATES_READER; -@RunWith(SpringRunner.class) +@ExtendWith(SpringExtension.class) @TestPropertySource("classpath:clustering-pipeline-test.properties") @ContextConfiguration(classes = {RSMergeAndSplitCandidatesReaderConfiguration.class, MongoTestConfiguration.class, BatchTestConfiguration.class}) -public class RSMergeAndSplitCandidatesReaderTest { +public class RSMergeAndSplitCandidatesReaderTest extends MongoTestContainerHelper { private static final String TEST_DB = "test-db"; @@ -61,9 +55,6 @@ public class RSMergeAndSplitCandidatesReaderTest { private static final String SUBMITTED_VARIANT_OPERATION_COLLECTION = "submittedVariantOperationEntity"; - @Autowired - private MongoClient mongoClient; - @Autowired private MongoTemplate mongoTemplate; @@ -75,18 +66,49 @@ public class RSMergeAndSplitCandidatesReaderTest { @Qualifier(RS_MERGE_CANDIDATES_READER) private MongoDbCursorItemReader rsMergeCandidatesReader; - //Required by nosql-unit - @Autowired - private ApplicationContext applicationContext; + @BeforeEach + public void setUp() { + mongoTemplate.getDb().drop(); + } + + @AfterEach + public void tearDown() { + mongoTemplate.getDb().drop(); + } + + @Test + public void readAllRSSplitCandidates() throws Exception { + createSplitCandidateEntries(); + List submittedVariantOperationEntities = new ArrayList<>(); + SubmittedVariantOperationEntity submittedVariantOperationEntity; + ExecutionContext executionContext = new ExecutionContext(); + this.rsSplitCandidatesReader.open(executionContext); + while ((submittedVariantOperationEntity = rsSplitCandidatesReader.read()) != null) { + submittedVariantOperationEntities.add(submittedVariantOperationEntity); + } + //Ensure that the reader reads the two operations created during the setup routine + assertEquals(2, submittedVariantOperationEntities.size()); + } - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName(TEST_DB).build()); - private SubmittedVariantEntity createSSWithLocus(Long ssAccession, Long rsAccession, Long start, String reference, + @Test + public void readAllRSMergeCandidates() throws Exception { + createMergeCandidateEntries(); + List submittedVariantOperationEntities = new ArrayList<>(); + SubmittedVariantOperationEntity submittedVariantOperationEntity; + ExecutionContext executionContext = new ExecutionContext(); + this.rsMergeCandidatesReader.open(executionContext); + while ((submittedVariantOperationEntity = rsMergeCandidatesReader.read()) != null) { + submittedVariantOperationEntities.add(submittedVariantOperationEntity); + } + //Ensure that the reader reads the two operations created during the setup routine + assertEquals(2, submittedVariantOperationEntities.size()); + } + + private SubmittedVariantEntity createSSWithLocus(Long ssAccession, Long rsAccession, Long start, String reference, String alternate) { return new SubmittedVariantEntity(ssAccession, "hash" + ssAccession, ASSEMBLY, 60711, - "PRJ1", "chr1", start, reference, alternate, rsAccession, false, false, false, - false, 1); + "PRJ1", "chr1", start, reference, alternate, rsAccession, false, false, false, + false, 1); } private void createSplitCandidateEntries() { @@ -101,8 +123,8 @@ private void createSplitCandidateEntries() { // Note the next null in accessionIdDestiny. We are not merging the submitted variant into // anything. We are updating the submitted variant, changing its rs field splitOperation1.fill(RSMergeAndSplitCandidatesReaderConfiguration.SPLIT_CANDIDATES_EVENT_TYPE, - ss1.getAccession(), null, "Hash mismatch with rs1", - Arrays.asList(ss1, ss2, ss3)); + ss1.getAccession(), null, "Hash mismatch with rs1", + Arrays.asList(ss1, ss2, ss3)); splitOperation1.setId(ClusteringWriter.getSplitCandidateId(splitOperation1)); SubmittedVariantInactiveEntity ss4 = new SubmittedVariantInactiveEntity( @@ -115,9 +137,9 @@ private void createSplitCandidateEntries() { // Note the next null in accessionIdDestiny. We are not merging the submitted variant into // anything. We are updating the submitted variant, changing its rs field splitOperation2.fill(RSMergeAndSplitCandidatesReaderConfiguration.SPLIT_CANDIDATES_EVENT_TYPE, - ss4.getAccession(), - null, "Hash mismatch with rs2", - Arrays.asList(ss4, ss5, ss6)); + ss4.getAccession(), + null, "Hash mismatch with rs2", + Arrays.asList(ss4, ss5, ss6)); splitOperation2.setId(ClusteringWriter.getSplitCandidateId(splitOperation2)); mongoTemplate.insert(splitOperation1, SUBMITTED_VARIANT_OPERATION_COLLECTION); @@ -136,8 +158,8 @@ private void createMergeCandidateEntries() { // Note the next null in accessionIdDestiny. We are not merging the submitted variant into // anything. We are updating the submitted variant, changing its rs field mergeOperation1.fill(RSMergeAndSplitCandidatesReaderConfiguration.MERGE_CANDIDATES_EVENT_TYPE, - ss1.getAccession(), null, "Different RS with matching loci", - Arrays.asList(ss1, ss2, ss3)); + ss1.getAccession(), null, "Different RS with matching loci", + Arrays.asList(ss1, ss2, ss3)); mergeOperation1.setId(ClusteringWriter.getMergeCandidateId(mergeOperation1)); SubmittedVariantInactiveEntity ss4 = new SubmittedVariantInactiveEntity( @@ -150,54 +172,12 @@ private void createMergeCandidateEntries() { // Note the next null in accessionIdDestiny. We are not merging the submitted variant into // anything. We are updating the submitted variant, changing its rs field mergeOperation2.fill(RSMergeAndSplitCandidatesReaderConfiguration.MERGE_CANDIDATES_EVENT_TYPE, - ss4.getAccession(), - null, "Different RS with matching loci", - Arrays.asList(ss4, ss5, ss6)); + ss4.getAccession(), + null, "Different RS with matching loci", + Arrays.asList(ss4, ss5, ss6)); mergeOperation2.setId(ClusteringWriter.getMergeCandidateId(mergeOperation2)); mongoTemplate.insert(mergeOperation1, SUBMITTED_VARIANT_OPERATION_COLLECTION); mongoTemplate.insert(mergeOperation2, SUBMITTED_VARIANT_OPERATION_COLLECTION); } - - @Before - public void setUp() { - cleanupDB(); - } - - @After - public void tearDown() { - cleanupDB(); - } - - private void cleanupDB() { - mongoClient.dropDatabase(TEST_DB); - } - - @Test - public void readAllRSSplitCandidates() throws Exception { - createSplitCandidateEntries(); - List submittedVariantOperationEntities = new ArrayList<>(); - SubmittedVariantOperationEntity submittedVariantOperationEntity; - ExecutionContext executionContext = new ExecutionContext(); - this.rsSplitCandidatesReader.open(executionContext); - while ((submittedVariantOperationEntity = rsSplitCandidatesReader.read()) != null) { - submittedVariantOperationEntities.add(submittedVariantOperationEntity); - } - //Ensure that the reader reads the two operations created during the setup routine - assertEquals(2, submittedVariantOperationEntities.size()); - } - - @Test - public void readAllRSMergeCandidates() throws Exception { - createMergeCandidateEntries(); - List submittedVariantOperationEntities = new ArrayList<>(); - SubmittedVariantOperationEntity submittedVariantOperationEntity; - ExecutionContext executionContext = new ExecutionContext(); - this.rsMergeCandidatesReader.open(executionContext); - while ((submittedVariantOperationEntity = rsMergeCandidatesReader.read()) != null) { - submittedVariantOperationEntities.add(submittedVariantOperationEntity); - } - //Ensure that the reader reads the two operations created during the setup routine - assertEquals(2, submittedVariantOperationEntities.size()); - } } \ No newline at end of file diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/RSMergeWriterTest.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/RSMergeWriterTest.java index 7533077fe..c18812865 100644 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/RSMergeWriterTest.java +++ b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/RSMergeWriterTest.java @@ -15,40 +15,32 @@ */ package uk.ac.ebi.eva.accession.clustering.batch.io; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; -import com.mongodb.MongoClient; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.batch.item.Chunk; import org.springframework.batch.item.ExecutionContext; import org.springframework.batch.item.ItemWriter; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.context.ApplicationContext; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.data.mongodb.core.query.Criteria; import org.springframework.data.mongodb.core.query.Query; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; -import uk.ac.ebi.ampt2d.commons.accession.core.exceptions.AccessionDeprecatedException; -import uk.ac.ebi.ampt2d.commons.accession.core.exceptions.AccessionDoesNotExistException; +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.ampt2d.commons.accession.core.exceptions.AccessionMergedException; import uk.ac.ebi.ampt2d.commons.accession.core.models.EventType; import uk.ac.ebi.ampt2d.commons.accession.hashing.SHA1HashingFunction; import uk.ac.ebi.ampt2d.commons.accession.persistence.mongodb.document.InactiveSubDocument; - import uk.ac.ebi.eva.accession.clustering.configuration.batch.io.RSMergeAndSplitCandidatesReaderConfiguration; import uk.ac.ebi.eva.accession.clustering.configuration.batch.io.RSMergeAndSplitWriterConfiguration; import uk.ac.ebi.eva.accession.clustering.metric.ClusteringMetric; import uk.ac.ebi.eva.accession.clustering.test.DatabaseState; import uk.ac.ebi.eva.accession.clustering.test.configuration.BatchTestConfiguration; import uk.ac.ebi.eva.accession.clustering.test.configuration.MongoTestConfiguration; -import uk.ac.ebi.eva.accession.clustering.test.rule.FixSpringMongoDbRule; import uk.ac.ebi.eva.accession.core.model.ClusteredVariant; import uk.ac.ebi.eva.accession.core.model.IClusteredVariant; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpClusteredVariantEntity; @@ -62,6 +54,7 @@ import uk.ac.ebi.eva.accession.core.service.nonhuman.ClusteredVariantAccessioningService; import uk.ac.ebi.eva.accession.core.service.nonhuman.SubmittedVariantAccessioningService; import uk.ac.ebi.eva.accession.core.summary.ClusteredVariantSummaryFunction; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.commons.core.models.pipeline.Variant; import uk.ac.ebi.eva.commons.mongodb.readers.MongoDbCursorItemReader; import uk.ac.ebi.eva.metrics.metric.MetricCompute; @@ -74,22 +67,20 @@ import java.util.stream.Collectors; import java.util.stream.Stream; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertThrows; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.CLUSTERED_CLUSTERING_WRITER; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.RS_MERGE_CANDIDATES_READER; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.RS_MERGE_WRITER; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.RS_SPLIT_CANDIDATES_READER; -@RunWith(SpringRunner.class) +@ExtendWith(SpringExtension.class) @TestPropertySource("classpath:merge-split-test.properties") @ContextConfiguration(classes = {RSMergeAndSplitCandidatesReaderConfiguration.class, RSMergeAndSplitWriterConfiguration.class, MongoTestConfiguration.class, BatchTestConfiguration.class}) -public class RSMergeWriterTest { - - private static final String TEST_DB = "test-db"; +public class RSMergeWriterTest extends MongoTestContainerHelper { private static final String ASSEMBLY = "GCA_000000001.1"; @@ -99,9 +90,6 @@ public class RSMergeWriterTest { private static final String SUBMITTED_VARIANT_OPERATION_COLLECTION = "submittedVariantOperationEntity"; - @Autowired - private MongoClient mongoClient; - @Autowired private MongoTemplate mongoTemplate; @@ -121,14 +109,6 @@ public class RSMergeWriterTest { @Qualifier(CLUSTERED_CLUSTERING_WRITER) private ClusteringWriter clusteringWriter; - //Required by nosql-unit - @Autowired - private ApplicationContext applicationContext; - - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName(TEST_DB).build()); - private SubmittedVariantEntity ss1, ss2, ss3, ss4, ss5, ss6, ss7, ss8, ss9; @Autowired @@ -156,8 +136,8 @@ private SubmittedVariantEntity createSS(Long ssAccession, Long rsAccession, Long String alternate, String assemblyToUse) { return new SubmittedVariantEntity(ssAccession, "hash" + ssAccession + assemblyToUse, assemblyToUse, 60711, - "PRJ1", "chr1", start, reference, alternate, rsAccession, false, false, false, - false, 1); + "PRJ1", "chr1", start, reference, alternate, rsAccession, false, false, false, + false, 1); } private void createMergeAndSplitCandidateEntries() { @@ -187,8 +167,8 @@ private void createMergeAndSplitCandidateEntries() { SubmittedVariantInactiveEntity ssInactive4 = new SubmittedVariantInactiveEntity(ss4); SubmittedVariantOperationEntity mergeOperation1 = new SubmittedVariantOperationEntity(); mergeOperation1.fill(RSMergeAndSplitCandidatesReaderConfiguration.MERGE_CANDIDATES_EVENT_TYPE, - ss1.getAccession(), null, "Different RS with matching loci", - Arrays.asList(ssInactive1, ssInactive4)); + ss1.getAccession(), null, "Different RS with matching loci", + Arrays.asList(ssInactive1, ssInactive4)); mergeOperation1.setId(ClusteringWriter.getMergeCandidateId(mergeOperation1)); //ss2 will be inserted to dbsnpSubmittedVariantEntity and ss5 to submittedVariantEntity collections respectively @@ -199,9 +179,9 @@ private void createMergeAndSplitCandidateEntries() { SubmittedVariantInactiveEntity ssInactive5 = new SubmittedVariantInactiveEntity(ss5); SubmittedVariantOperationEntity mergeOperation2 = new SubmittedVariantOperationEntity(); mergeOperation2.fill(RSMergeAndSplitCandidatesReaderConfiguration.MERGE_CANDIDATES_EVENT_TYPE, - ss2.getAccession(), - null, "Different RS with matching loci", - Arrays.asList(ssInactive2, ssInactive5)); + ss2.getAccession(), + null, "Different RS with matching loci", + Arrays.asList(ssInactive2, ssInactive5)); mergeOperation2.setId(ClusteringWriter.getMergeCandidateId(mergeOperation2)); //Candidates for split are entries with same RS but different locus @@ -218,16 +198,16 @@ private void createMergeAndSplitCandidateEntries() { SubmittedVariantOperationEntity splitOperation2 = new SubmittedVariantOperationEntity(); SubmittedVariantOperationEntity splitOperation3 = new SubmittedVariantOperationEntity(); splitOperation1.fill(RSMergeAndSplitCandidatesReaderConfiguration.SPLIT_CANDIDATES_EVENT_TYPE, - ss2.getAccession(), "Hash mismatch with " + ss2.getClusteredVariantAccession(), - Arrays.asList(ssInactive2, ssInactive8)); + ss2.getAccession(), "Hash mismatch with " + ss2.getClusteredVariantAccession(), + Arrays.asList(ssInactive2, ssInactive8)); splitOperation1.setId(ClusteringWriter.getSplitCandidateId(splitOperation1)); splitOperation2.fill(RSMergeAndSplitCandidatesReaderConfiguration.SPLIT_CANDIDATES_EVENT_TYPE, - ss4.getAccession(), "Hash mismatch with " + ss4.getClusteredVariantAccession(), - Arrays.asList(ssInactive4, ssInactive6, ssInactive7)); + ss4.getAccession(), "Hash mismatch with " + ss4.getClusteredVariantAccession(), + Arrays.asList(ssInactive4, ssInactive6, ssInactive7)); splitOperation2.setId(ClusteringWriter.getSplitCandidateId(splitOperation2)); splitOperation3.fill(RSMergeAndSplitCandidatesReaderConfiguration.SPLIT_CANDIDATES_EVENT_TYPE, - ss5.getAccession(), "Hash mismatch with " + ss5.getClusteredVariantAccession(), - Arrays.asList(ssInactive5, ssInactive9)); + ss5.getAccession(), "Hash mismatch with " + ss5.getClusteredVariantAccession(), + Arrays.asList(ssInactive5, ssInactive9)); splitOperation3.setId(ClusteringWriter.getSplitCandidateId(splitOperation3)); List ssToInsertToDbsnpSVE = Arrays.asList(ss1, ss2); @@ -236,33 +216,29 @@ private void createMergeAndSplitCandidateEntries() { mongoTemplate.insert(ssToInsertToDbsnpSVE, DBSNP_SUBMITTED_VARIANT_COLLECTION); mongoTemplate.insert(ssToInsertToSVE, SUBMITTED_VARIANT_COLLECTION); mongoTemplate.insert(Arrays.asList(mergeOperation1, mergeOperation2, - splitOperation1, splitOperation2, splitOperation3), - SUBMITTED_VARIANT_OPERATION_COLLECTION); - } - - private void cleanup() { - mongoClient.dropDatabase(TEST_DB); - metricCompute.clearCount(); + splitOperation1, splitOperation2, splitOperation3), + SUBMITTED_VARIANT_OPERATION_COLLECTION); } - @Before + @BeforeEach public void setUp() { - cleanup(); + mongoTemplate.getDb().drop(); + metricCompute.clearCount(); } - @After + @AfterEach public void tearDown() { - cleanup(); + mongoTemplate.getDb().drop(); + metricCompute.clearCount(); } - private void assertRSAssociatedWithSS(Long rsAccession, SubmittedVariantEntity submittedVariantEntity) - throws AccessionDoesNotExistException, AccessionMergedException, AccessionDeprecatedException { + private void assertRSAssociatedWithSS(Long rsAccession, SubmittedVariantEntity submittedVariantEntity) { assertEquals(rsAccession, - submittedVariantAccessioningService - .getAllActiveByAssemblyAndAccessionIn(ASSEMBLY, - Collections.singletonList( - submittedVariantEntity.getAccession())) - .stream().findFirst().get().getData().getClusteredVariantAccession()); + submittedVariantAccessioningService + .getAllActiveByAssemblyAndAccessionIn(ASSEMBLY, + Collections.singletonList( + submittedVariantEntity.getAccession())) + .stream().findFirst().get().getData().getClusteredVariantAccession()); } @Test @@ -294,24 +270,28 @@ public void testWriteRSMerges() throws Exception { } //Perform merge - rsMergeWriter.write(submittedVariantOperationEntities); + rsMergeWriter.write(new Chunk<>(submittedVariantOperationEntities)); //Ensure operations collections have appropriate entries to reflect rs4 merged to rs1 & rs5 merged to rs2 assertEquals(Long.valueOf(1L), clusteredVariantOperationRepository.findAllByAccession(4L).get(0) - .getMergedInto()); + .getMergedInto()); assertEquals(Long.valueOf(2L), clusteredVariantOperationRepository.findAllByAccession(5L).get(0) - .getMergedInto()); - assertEquals ("Original rs4 associated with SS was merged into rs1.", - submittedVariantOperationRepository.findAllByAccession(4L).get(0).getReason()); + .getMergedInto()); + assertEquals("Original rs4 associated with SS was merged into rs1.", + submittedVariantOperationRepository.findAllByAccession(4L).get(0).getReason()); assertEquals("Original rs5 associated with SS was merged into rs2.", - submittedVariantOperationRepository.findAllByAccession(5L).get(0).getReason()); + submittedVariantOperationRepository.findAllByAccession(5L).get(0).getReason()); // Ensure that RS1 and RS2 are present in the clustered variant collection post merge assertEquals(Long.valueOf(1L), clusteredVariantAccessioningService.getByAccession(1L).getAccession()); assertEquals(Long.valueOf(2L), clusteredVariantAccessioningService.getByAccession(2L).getAccession()); - assertThrows(AccessionMergedException.class, () -> {clusteredVariantAccessioningService.getByAccession(4L);}); - assertThrows(AccessionMergedException.class, () -> {clusteredVariantAccessioningService.getByAccession(5L);}); + assertThrows(AccessionMergedException.class, () -> { + clusteredVariantAccessioningService.getByAccession(4L); + }); + assertThrows(AccessionMergedException.class, () -> { + clusteredVariantAccessioningService.getByAccession(5L); + }); //After merge SS-RS associations: rs4 merged to rs1 & rs5 merged to rs2 /* @@ -337,13 +317,13 @@ public void testWriteRSMerges() throws Exception { // After rs4 merge to rs1 and rs5 merge to rs2 // split candidates operations involving rs4 and rs5 should no longer be present assertFalse(submittedVariantOperationRepository - .findAllByAccession(4L).stream() - .anyMatch(event -> event.getEventType().equals( - RSMergeAndSplitCandidatesReaderConfiguration.SPLIT_CANDIDATES_EVENT_TYPE))); + .findAllByAccession(4L).stream() + .anyMatch(event -> event.getEventType().equals( + RSMergeAndSplitCandidatesReaderConfiguration.SPLIT_CANDIDATES_EVENT_TYPE))); assertFalse(submittedVariantOperationRepository - .findAllByAccession(5L).stream() - .anyMatch(event -> event.getEventType().equals( - RSMergeAndSplitCandidatesReaderConfiguration.SPLIT_CANDIDATES_EVENT_TYPE))); + .findAllByAccession(5L).stream() + .anyMatch(event -> event.getEventType().equals( + RSMergeAndSplitCandidatesReaderConfiguration.SPLIT_CANDIDATES_EVENT_TYPE))); // Post-merge SS1, SS4, SS6 and SS7 all share rs1 but have different locus // and hence generate one split candidate event // Post-merge SS2, SS5, SS8 and SS9 all share rs2 but have different locus @@ -352,7 +332,7 @@ public void testWriteRSMerges() throws Exception { SubmittedVariantOperationEntity tempObj; rsSplitCandidatesReader.open(new ExecutionContext()); while ((tempObj = rsSplitCandidatesReader.read()) != null) { - splitEvents.add(tempObj); + splitEvents.add(tempObj); } assertSplitCandidateEvents(1L, Arrays.asList(1L, 4L, 6L, 7L), splitEvents); assertSplitCandidateEvents(2L, Arrays.asList(2L, 5L, 8L, 9L), splitEvents); @@ -369,10 +349,10 @@ public void testWriteRSMerges() throws Exception { // Check if a given RS has the participating SS listed in the split candidates event private void assertSplitCandidateEvents(Long rsAccession, List expectedParticipatingSS, - List splitEvents) throws Exception { + List splitEvents) { splitEvents = splitEvents.stream().filter(e -> e.getInactiveObjects().get(0) - .getClusteredVariantAccession().equals(rsAccession)) - .collect(Collectors.toList()); + .getClusteredVariantAccession().equals(rsAccession)) + .collect(Collectors.toList()); assertEquals(1, splitEvents.size()); SubmittedVariantOperationEntity splitEvent = splitEvents.get(0); assertEquals("Hash mismatch with " + rsAccession, splitEvent.getReason()); @@ -380,12 +360,12 @@ private void assertSplitCandidateEvents(Long rsAccession, List expectedPar assertEquals(expectedParticipatingSS.size(), participatingSSInSplitEvent.size()); assertTrue(participatingSSInSplitEvent.stream() - .map(InactiveSubDocument::getAccession) - .collect(Collectors.toSet()) - .containsAll(expectedParticipatingSS)); + .map(InactiveSubDocument::getAccession) + .collect(Collectors.toSet()) + .containsAll(expectedParticipatingSS)); // Ensure that all the inactive objects have the same RS assertFalse(participatingSSInSplitEvent.stream() - .anyMatch(e -> !e.getClusteredVariantAccession().equals(rsAccession))); + .anyMatch(e -> !e.getClusteredVariantAccession().equals(rsAccession))); } @Test @@ -411,7 +391,7 @@ public void testMultiLevelRSMerges() throws Exception { } //Perform merge - rsMergeWriter.write(submittedVariantOperationEntities); + rsMergeWriter.write(new Chunk<>(submittedVariantOperationEntities)); //After merge SS-RS associations: rs2 merged to rs1, Merge target for rs3 inferred as: rs3 -> rs2 -> rs1 /* @@ -458,7 +438,7 @@ public void testRSWithSameHashAsMergeDestinationAlreadyExistsInDBWithHigherAcces } //Perform merge - rsMergeWriter.write(submittedVariantOperationEntities); + rsMergeWriter.write(new Chunk<>(submittedVariantOperationEntities)); // Check rs2,rs3 merged into to rs1 assertMergeOp(ss2.getClusteredVariantAccession(), ss1.getClusteredVariantAccession(), ASSEMBLY); @@ -496,7 +476,7 @@ public void testRSWithSameHashAsMergeDestinationAlreadyExistsInDBWithLowerAccess } //Perform merge - rsMergeWriter.write(submittedVariantOperationEntities); + rsMergeWriter.write(new Chunk<>(submittedVariantOperationEntities)); // Check rs2,rs3 merged into to rs1 assertMergeOp(ss2.getClusteredVariantAccession(), ss1.getClusteredVariantAccession(), ASSEMBLY); @@ -512,9 +492,9 @@ private void assertMergeOp(Long mergee, Long merger, String assemblyToUse) { assertEquals(0, this.clusteredVariantAccessioningService .getAllActiveByAssemblyAndAccessionIn(assemblyToUse, Arrays.asList(mergee)).size()); assertEquals(1, this.mongoTemplate.find(Query.query(Criteria.where("inactiveObjects.asm") - .is(assemblyToUse).and("accession").is(mergee) - .and("mergeInto").is(merger)), - DbsnpClusteredVariantOperationEntity.class).size()); + .is(assemblyToUse).and("accession").is(mergee) + .and("mergeInto").is(merger)), + DbsnpClusteredVariantOperationEntity.class).size()); } public void createMultiLevelMergeScenario() { @@ -528,23 +508,23 @@ public void createMultiLevelMergeScenario() { SubmittedVariantOperationEntity mergeOperation1 = new SubmittedVariantOperationEntity(); mergeOperation1.fill(RSMergeAndSplitCandidatesReaderConfiguration.MERGE_CANDIDATES_EVENT_TYPE, - ss1.getAccession(), null, "Different RS with matching loci", - Stream.of(ss1, ss2).map(SubmittedVariantInactiveEntity::new).collect(Collectors.toList())); + ss1.getAccession(), null, "Different RS with matching loci", + Stream.of(ss1, ss2).map(SubmittedVariantInactiveEntity::new).collect(Collectors.toList())); mergeOperation1.setId(ClusteringWriter.getMergeCandidateId(mergeOperation1)); SubmittedVariantOperationEntity mergeOperation2 = new SubmittedVariantOperationEntity(); mergeOperation2.fill(RSMergeAndSplitCandidatesReaderConfiguration.MERGE_CANDIDATES_EVENT_TYPE, - ss3.getAccession(), null, "Different RS with matching loci", - Stream.of(ss3, ss4).map(SubmittedVariantInactiveEntity::new).collect(Collectors.toList())); + ss3.getAccession(), null, "Different RS with matching loci", + Stream.of(ss3, ss4).map(SubmittedVariantInactiveEntity::new).collect(Collectors.toList())); mergeOperation2.setId(ClusteringWriter.getMergeCandidateId(mergeOperation2)); SubmittedVariantOperationEntity splitOperation1 = new SubmittedVariantOperationEntity(); splitOperation1.fill(RSMergeAndSplitCandidatesReaderConfiguration.SPLIT_CANDIDATES_EVENT_TYPE, - ss2.getAccession(), null, "Hash mismatch with " + ss2.getClusteredVariantAccession(), - Stream.of(ss2, ss3).map(SubmittedVariantInactiveEntity::new).collect(Collectors.toList())); + ss2.getAccession(), null, "Hash mismatch with " + ss2.getClusteredVariantAccession(), + Stream.of(ss2, ss3).map(SubmittedVariantInactiveEntity::new).collect(Collectors.toList())); splitOperation1.setId(ClusteringWriter.getSplitCandidateId(splitOperation1)); this.mongoTemplate.insert(Arrays.asList(mergeOperation1, mergeOperation2, splitOperation1), - SUBMITTED_VARIANT_OPERATION_COLLECTION); + SUBMITTED_VARIANT_OPERATION_COLLECTION); } private void createSimilarEntriesInAnotherAssembly() { @@ -555,7 +535,7 @@ private void createSimilarEntriesInAnotherAssembly() { SubmittedVariantEntity ss3_another_asm = createSS(3L, 2L, 103L, "A", "G", "ASM_ANOTHER"); SubmittedVariantEntity ss4_another_asm = createSS(4L, 3L, 103L, "T", "C", "ASM_ANOTHER"); this.mongoTemplate.insert(Arrays.asList(ss1_another_asm, ss2_another_asm, ss3_another_asm), - DBSNP_SUBMITTED_VARIANT_COLLECTION); + DBSNP_SUBMITTED_VARIANT_COLLECTION); this.mongoTemplate.insert(Collections.singletonList(ss4_another_asm), SUBMITTED_VARIANT_COLLECTION); DbsnpClusteredVariantEntity rs1_another_asm = this.createRS(ss1_another_asm); @@ -563,8 +543,8 @@ private void createSimilarEntriesInAnotherAssembly() { // see https://www.ebi.ac.uk/panda/jira/browse/EVA-2974?focusedCommentId=405919&page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel#comment-405919 DbsnpClusteredVariantOperationEntity rs2ToRs1MergeOp = new DbsnpClusteredVariantOperationEntity(); rs2ToRs1MergeOp.fill(EventType.MERGED, 2L, 1L, - "Identical clustered variant received multiple RS identifiers", - Arrays.asList(new DbsnpClusteredVariantInactiveEntity(rs1_another_asm))); + "Identical clustered variant received multiple RS identifiers", + Arrays.asList(new DbsnpClusteredVariantInactiveEntity(rs1_another_asm))); this.mongoTemplate.save(rs2ToRs1MergeOp); } @@ -589,11 +569,11 @@ public void testIdempotentMergeWrites() throws Exception { } //First run of RS Merges - rsMergeWriter.write(submittedVariantOperationEntities); + rsMergeWriter.write(new Chunk<>(submittedVariantOperationEntities)); DatabaseState databaseStateAfterFirstMergeWrite = DatabaseState.getCurrentDatabaseState(this.mongoTemplate); - rsMergeWriter.write(submittedVariantOperationEntities); + rsMergeWriter.write(new Chunk<>(submittedVariantOperationEntities)); DatabaseState databaseStateAfterSecondMergeWrite = DatabaseState.getCurrentDatabaseState(this.mongoTemplate); - rsMergeWriter.write(submittedVariantOperationEntities); + rsMergeWriter.write(new Chunk<>(submittedVariantOperationEntities)); DatabaseState databaseStateAfterThirdMergeWrite = DatabaseState.getCurrentDatabaseState(this.mongoTemplate); assertEquals(databaseStateAfterSecondMergeWrite, databaseStateAfterFirstMergeWrite); @@ -601,15 +581,15 @@ public void testIdempotentMergeWrites() throws Exception { } private DbsnpClusteredVariantEntity createRS(SubmittedVariantEntity sve) { - Function hashingFunction = new ClusteredVariantSummaryFunction().andThen( + Function hashingFunction = new ClusteredVariantSummaryFunction().andThen( new SHA1HashingFunction()); ClusteredVariant cv = new ClusteredVariant(sve.getReferenceSequenceAccession(), sve.getTaxonomyAccession(), - sve.getContig(), - sve.getStart(), - new Variant(sve.getContig(), sve.getStart(), sve.getStart(), - sve.getReferenceAllele(), - sve.getAlternateAllele()).getType(), - true, null); + sve.getContig(), + sve.getStart(), + new Variant(sve.getContig(), sve.getStart(), sve.getStart(), + sve.getReferenceAllele(), + sve.getAlternateAllele()).getType(), + true, null); String hash = hashingFunction.apply(cv); return new DbsnpClusteredVariantEntity(sve.getClusteredVariantAccession(), hash, cv); } diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/RSSplitWriterTest.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/RSSplitWriterTest.java index 29b6054f8..e2c326179 100644 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/RSSplitWriterTest.java +++ b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/RSSplitWriterTest.java @@ -15,37 +15,29 @@ */ package uk.ac.ebi.eva.accession.clustering.batch.io; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; -import com.mongodb.MongoClient; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; -import org.junit.runner.RunWith; - +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mockito; import org.springframework.batch.core.JobExecution; +import org.springframework.batch.item.Chunk; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.test.mock.mockito.MockBean; -import org.springframework.context.ApplicationContext; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.ampt2d.commons.accession.core.models.AccessionWrapper; import uk.ac.ebi.ampt2d.commons.accession.hashing.SHA1HashingFunction; - import uk.ac.ebi.eva.accession.clustering.configuration.batch.io.RSMergeAndSplitCandidatesReaderConfiguration; import uk.ac.ebi.eva.accession.clustering.configuration.batch.io.RSMergeAndSplitWriterConfiguration; import uk.ac.ebi.eva.accession.clustering.metric.ClusteringMetric; import uk.ac.ebi.eva.accession.clustering.test.DatabaseState; import uk.ac.ebi.eva.accession.clustering.test.configuration.BatchTestConfiguration; import uk.ac.ebi.eva.accession.clustering.test.configuration.MongoTestConfiguration; -import uk.ac.ebi.eva.accession.clustering.test.rule.FixSpringMongoDbRule; import uk.ac.ebi.eva.accession.core.model.ISubmittedVariant; import uk.ac.ebi.eva.accession.core.model.SubmittedVariant; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpClusteredVariantEntity; @@ -56,6 +48,7 @@ import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantOperationEntity; import uk.ac.ebi.eva.accession.core.service.nonhuman.SubmittedVariantAccessioningService; import uk.ac.ebi.eva.accession.core.summary.SubmittedVariantSummaryFunction; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.metrics.metric.MetricCompute; import java.io.File; @@ -68,28 +61,20 @@ import java.util.stream.Collectors; import java.util.stream.Stream; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.CLUSTERED_CLUSTERING_WRITER; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.RS_SPLIT_WRITER; -@RunWith(SpringRunner.class) +@ExtendWith(SpringExtension.class) @TestPropertySource("classpath:merge-split-test.properties") -@ContextConfiguration(classes = {RSMergeAndSplitCandidatesReaderConfiguration.class, RSMergeAndSplitWriterConfiguration.class, - MongoTestConfiguration.class, BatchTestConfiguration.class}) -public class RSSplitWriterTest { - - private static final String TEST_DB = "test-db"; +@ContextConfiguration(classes = {RSMergeAndSplitCandidatesReaderConfiguration.class, + RSMergeAndSplitWriterConfiguration.class, MongoTestConfiguration.class, BatchTestConfiguration.class}) +public class RSSplitWriterTest extends MongoTestContainerHelper { private static final String ASSEMBLY = "GCA_000000001.1"; - @Rule - public final ExpectedException thrown = ExpectedException.none(); - - @Autowired - private MongoClient mongoClient; - @Autowired private MongoTemplate mongoTemplate; @@ -107,14 +92,6 @@ public class RSSplitWriterTest { @MockBean private JobExecution jobExecution; - //Required by nosql-unit - @Autowired - private ApplicationContext applicationContext; - - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName(TEST_DB).build()); - private SubmittedVariantEntity ss1, ss2, ss3, ss4, ss5, ss6; @Autowired @@ -123,22 +100,20 @@ public class RSSplitWriterTest { @Autowired private MetricCompute metricCompute; - @Before + @BeforeEach public void setUp() throws IOException { - cleanup(); + mongoTemplate.getDb().drop(); + metricCompute.clearCount(); + Files.deleteIfExists(this.rsReportFile.toPath()); Mockito.when(jobExecution.getJobId()).thenReturn(1L); rsSplitWriter.setJobExecution(jobExecution); clusteringWriter.setJobExecution(jobExecution); } - @After + @AfterEach public void tearDown() throws IOException { - cleanup(); - } - - private void cleanup() throws IOException { - mongoClient.dropDatabase(TEST_DB); + mongoTemplate.getDb().drop(); metricCompute.clearCount(); Files.deleteIfExists(this.rsReportFile.toPath()); } @@ -148,7 +123,7 @@ private SubmittedVariantEntity createSS(Long ssAccession, Long rsAccession, Long Function hashingFunction = new SubmittedVariantSummaryFunction().andThen( new SHA1HashingFunction()); SubmittedVariant temp = new SubmittedVariant(ASSEMBLY, 60711, "PRJ1", "chr1", start, reference, alternate, - rsAccession); + rsAccession); return new SubmittedVariantEntity(ssAccession, hashingFunction.apply(temp), temp, 1); } @@ -163,29 +138,29 @@ public void testSplitsWithUnequalNumberOfDistinctHashes() throws Exception { // Create multiple entries in clustered variant collection - one for each distinct loci but with the same accession List multipleEntriesWithSameRS = Stream.of(ss1, ss2, ss4) - .map(clusteringWriter::toClusteredVariantEntity) - .collect(Collectors.toList()); + .map(clusteringWriter::toClusteredVariantEntity) + .collect(Collectors.toList()); mongoTemplate.insert(Arrays.asList(ss1, ss2, ss3), DbsnpSubmittedVariantEntity.class); mongoTemplate.insert(Collections.singletonList(ss4), SubmittedVariantEntity.class); mongoTemplate.insert(multipleEntriesWithSameRS, DbsnpClusteredVariantEntity.class); SubmittedVariantOperationEntity splitOperation = new SubmittedVariantOperationEntity(); splitOperation.fill(RSMergeAndSplitCandidatesReaderConfiguration.SPLIT_CANDIDATES_EVENT_TYPE, - ss1.getAccession(), "Hash mismatch with " + ss1.getClusteredVariantAccession(), - Stream.of(ss1, ss2, ss3, ss4).map(SubmittedVariantInactiveEntity::new) - .collect(Collectors.toList()) + ss1.getAccession(), "Hash mismatch with " + ss1.getClusteredVariantAccession(), + Stream.of(ss1, ss2, ss3, ss4).map(SubmittedVariantInactiveEntity::new) + .collect(Collectors.toList()) ); mongoTemplate.insert(Collections.singletonList(splitOperation), SubmittedVariantOperationEntity.class); - rsSplitWriter.write(Collections.singletonList(splitOperation)); + rsSplitWriter.write(Chunk.of(splitOperation)); // Per splitting policy, ensure that SS2 and SS3 get to retain the old RS // because the RS hash for start position 101 is supported in these 2 variants List> ssAssociatedWithRS1 = - submittedVariantAccessioningService.getByClusteredVariantAccessionIn( - Collections.singletonList(sameRSAccessionToBeUsedForDifferentLoci)); + submittedVariantAccessioningService.getByClusteredVariantAccessionIn( + Collections.singletonList(sameRSAccessionToBeUsedForDifferentLoci)); assertEquals(2, ssAssociatedWithRS1.size()); assertTrue(ssAssociatedWithRS1.stream().map(AccessionWrapper::getAccession).collect(Collectors.toSet()) - .containsAll(Arrays.asList(ss2.getAccession(), ss3.getAccession()))); + .containsAll(Arrays.asList(ss2.getAccession(), ss3.getAccession()))); /* Check clustering counts */ // 2 new RS IDs created for hashes with start positions 100 and 102 assertEquals(2, metricCompute.getCount(ClusteringMetric.CLUSTERED_VARIANTS_CREATED)); @@ -200,9 +175,9 @@ public void testSplitsWithUnequalNumberOfDistinctHashes() throws Exception { // RS ID created for a given hash in not deterministic during the split. // So, use regex to check for just the hashes in the RS report. assertTrue(rsReportLines.stream() - .anyMatch(line -> line.matches(".*\t20610F413155992F45A41FB1DC6005B2FAA1565B"))); + .anyMatch(line -> line.matches(".*\t20610F413155992F45A41FB1DC6005B2FAA1565B"))); assertTrue(rsReportLines.stream() - .anyMatch(line -> line.matches(".*\t717DE940ADEB9E9DCAF09600065BAAF9A91EE713"))); + .anyMatch(line -> line.matches(".*\t717DE940ADEB9E9DCAF09600065BAAF9A91EE713"))); } @Test @@ -218,20 +193,20 @@ public void testSplitsWithEqualNumberOfDistinctHashes() throws Exception { // Create multiple entries in clustered variant collection - one for each distinct loci but with the same accession List multipleEntriesWithSameRS = Stream.of(ss1, ss3, ss5) - .map(clusteringWriter::toClusteredVariantEntity) - .collect(Collectors.toList()); + .map(clusteringWriter::toClusteredVariantEntity) + .collect(Collectors.toList()); mongoTemplate.insert(Arrays.asList(ss1, ss2, ss3), DbsnpSubmittedVariantEntity.class); mongoTemplate.insert(Arrays.asList(ss4, ss5, ss6), SubmittedVariantEntity.class); mongoTemplate.insert(multipleEntriesWithSameRS, DbsnpClusteredVariantEntity.class); SubmittedVariantOperationEntity splitOperation = new SubmittedVariantOperationEntity(); splitOperation.fill(RSMergeAndSplitCandidatesReaderConfiguration.SPLIT_CANDIDATES_EVENT_TYPE, - ss1.getAccession(), "Hash mismatch with " + ss1.getClusteredVariantAccession(), - Stream.of(ss1, ss2, ss3, ss4, ss5, ss6).map(SubmittedVariantInactiveEntity::new) - .collect(Collectors.toList()) + ss1.getAccession(), "Hash mismatch with " + ss1.getClusteredVariantAccession(), + Stream.of(ss1, ss2, ss3, ss4, ss5, ss6).map(SubmittedVariantInactiveEntity::new) + .collect(Collectors.toList()) ); mongoTemplate.insert(Collections.singletonList(splitOperation), SubmittedVariantOperationEntity.class); - rsSplitWriter.write(Collections.singletonList(splitOperation)); + rsSplitWriter.write(Chunk.of(splitOperation)); // Per splitting policy, ensure that SS1 and SS2 get to retain the old RS // Even though locus with start 100, 101 and 102 have equal number of supporting variants @@ -241,7 +216,7 @@ public void testSplitsWithEqualNumberOfDistinctHashes() throws Exception { Collections.singletonList(sameRSAccessionToBeUsedForDifferentLoci)); assertEquals(2, ssAssociatedWithRS1.size()); assertTrue(ssAssociatedWithRS1.stream().map(AccessionWrapper::getAccession).collect(Collectors.toSet()) - .containsAll(Arrays.asList(ss1.getAccession(), ss2.getAccession()))); + .containsAll(Arrays.asList(ss1.getAccession(), ss2.getAccession()))); /* Check clustering counts */ // 2 entries created for hashes with start positions 100 and 102 assertEquals(2, metricCompute.getCount(ClusteringMetric.CLUSTERED_VARIANTS_CREATED)); @@ -256,9 +231,9 @@ public void testSplitsWithEqualNumberOfDistinctHashes() throws Exception { // RS ID created for a given hash in not deterministic during the split. // So, use regex to check for just the hashes in the RS report. assertTrue(rsReportLines.stream() - .anyMatch(line -> line.matches(".*\t717DE940ADEB9E9DCAF09600065BAAF9A91EE713"))); + .anyMatch(line -> line.matches(".*\t717DE940ADEB9E9DCAF09600065BAAF9A91EE713"))); assertTrue(rsReportLines.stream() - .anyMatch(line -> line.matches(".*\t0304A4760D6365E3BAA8E203AF0355248C05979D"))); + .anyMatch(line -> line.matches(".*\t0304A4760D6365E3BAA8E203AF0355248C05979D"))); } @Test @@ -269,23 +244,23 @@ public void testSplitsInvolvingSameSSWithSameRS_Scenario1() throws Exception { ss2 = createSS(2L, rs1Accession, 101L, "T", "A"); // Another SS with the same accession as SS2 but different REF/ALT SubmittedVariantEntity ss2_another = createSS(ss2.getAccession(), rs1Accession, 102L, - "T", ""); + "T", ""); // Create multiple entries in clustered variant collection - one for each distinct loci but with the same accession List multipleEntriesWithSameRS = Stream.of(ss1, ss2, ss2_another) - .map(clusteringWriter::toClusteredVariantEntity) - .collect(Collectors.toList()); + .map(clusteringWriter::toClusteredVariantEntity) + .collect(Collectors.toList()); mongoTemplate.insert(Arrays.asList(ss1, ss2, ss2_another), DbsnpSubmittedVariantEntity.class); mongoTemplate.insert(multipleEntriesWithSameRS, DbsnpClusteredVariantEntity.class); SubmittedVariantOperationEntity splitOperation = new SubmittedVariantOperationEntity(); splitOperation.fill(RSMergeAndSplitCandidatesReaderConfiguration.SPLIT_CANDIDATES_EVENT_TYPE, - ss1.getAccession(), "Hash mismatch with " + ss1.getClusteredVariantAccession(), - Stream.of(ss1, ss2, ss2_another).map(SubmittedVariantInactiveEntity::new) - .collect(Collectors.toList()) + ss1.getAccession(), "Hash mismatch with " + ss1.getClusteredVariantAccession(), + Stream.of(ss1, ss2, ss2_another).map(SubmittedVariantInactiveEntity::new) + .collect(Collectors.toList()) ); mongoTemplate.insert(Collections.singletonList(splitOperation), SubmittedVariantOperationEntity.class); - rsSplitWriter.write(Collections.singletonList(splitOperation)); + rsSplitWriter.write(Chunk.of(splitOperation)); // Even though both SS2 and SS2_another happen to have the same ID and the same RS but different locus // (can happen, see https://www.ebi.ac.uk/panda/jira/browse/EVA-2630), @@ -293,13 +268,13 @@ public void testSplitsInvolvingSameSSWithSameRS_Scenario1() throws Exception { // Here SS1-rsLocus1 gets to keep the old RS ID because SS1 is the older SS ID // SS2 and SS2_another get two different newly issued RS IDs respectively even though they have the same accession Long newRSForSS1 = submittedVariantAccessioningService.get(Collections.singletonList(ss1)).get(0).getData() - .getClusteredVariantAccession(); + .getClusteredVariantAccession(); Long newRSForSS2 = submittedVariantAccessioningService.get(Collections.singletonList(ss2)).get(0).getData() - .getClusteredVariantAccession(); + .getClusteredVariantAccession(); Long newRSForSS2_another = submittedVariantAccessioningService.get(Collections.singletonList(ss2_another)) - .get(0) - .getData() - .getClusteredVariantAccession(); + .get(0) + .getData() + .getClusteredVariantAccession(); assertEquals(rs1Accession, newRSForSS1); assertNotEquals(rs1Accession, newRSForSS2); assertNotEquals(rs1Accession, newRSForSS2_another); @@ -310,9 +285,9 @@ public void testSplitsInvolvingSameSSWithSameRS_Scenario1() throws Exception { // RS ID created for a given hash in not deterministic during the split. // So, use regex to check for just the hashes in the RS report. assertTrue(rsReportLines.stream() - .anyMatch(line -> line.matches(".*\t15F833E1CD27F19EEC66BE89E96B89C99D46A5AB"))); + .anyMatch(line -> line.matches(".*\t15F833E1CD27F19EEC66BE89E96B89C99D46A5AB"))); assertTrue(rsReportLines.stream() - .anyMatch(line -> line.matches(".*\t0304A4760D6365E3BAA8E203AF0355248C05979D"))); + .anyMatch(line -> line.matches(".*\t0304A4760D6365E3BAA8E203AF0355248C05979D"))); } @Test @@ -322,10 +297,10 @@ public void testSplitsInvolvingSameSSWithSameRS_Scenario2() throws Exception { ss1 = createSS(1L, rs1Accession, 100L, "C", "T"); // Another SS with the same accession as SS1 but different REF/ALT SubmittedVariantEntity ss1_another = createSS(ss1.getAccession(), rs1Accession, 101L, - "T", ""); + "T", ""); SubmittedVariantOperationEntity splitOperation = createScenario2_SplitsInvolvingSameSSWithSameRS(ss1, ss1_another, rs1Accession); - rsSplitWriter.write(Collections.singletonList(splitOperation)); + rsSplitWriter.write(Chunk.of(splitOperation)); // Even though the number of supporting variants for rsLocus2 (ss1) and rsLocus3 (ss1_another) are same @@ -336,10 +311,10 @@ public void testSplitsInvolvingSameSSWithSameRS_Scenario2() throws Exception { // RS locus representation of SS1_another - 1_chr1_101_DEL // See ClusteredVariantSplittingPolicy.java Long newRSForSS1 = submittedVariantAccessioningService.get(Collections.singletonList(ss1)).get(0).getData() - .getClusteredVariantAccession(); + .getClusteredVariantAccession(); Long newRSForSS1_another = submittedVariantAccessioningService.get(Collections.singletonList(ss1_another)) - .get(0).getData() - .getClusteredVariantAccession(); + .get(0).getData() + .getClusteredVariantAccession(); assertEquals(rs1Accession, newRSForSS1); assertNotEquals(rs1Accession, newRSForSS1_another); @@ -354,16 +329,16 @@ public void testSplitsInvolvingSameSSWithSameRS_Scenario2() throws Exception { (SubmittedVariantEntity ss1, SubmittedVariantEntity ss1_another, Long rs1Accession) { // Create multiple entries in clustered variant collection - one for each distinct loci but with the same accession List multipleEntriesWithSameRS = Stream.of(ss1, ss1_another) - .map(clusteringWriter::toClusteredVariantEntity) - .collect(Collectors.toList()); + .map(clusteringWriter::toClusteredVariantEntity) + .collect(Collectors.toList()); mongoTemplate.insert(Arrays.asList(ss1, ss1_another), DbsnpSubmittedVariantEntity.class); mongoTemplate.insert(multipleEntriesWithSameRS, DbsnpClusteredVariantEntity.class); SubmittedVariantOperationEntity splitOperation = new SubmittedVariantOperationEntity(); splitOperation.fill(RSMergeAndSplitCandidatesReaderConfiguration.SPLIT_CANDIDATES_EVENT_TYPE, - ss1.getAccession(), "Hash mismatch with " + ss1.getClusteredVariantAccession(), - Stream.of(ss1, ss1_another).map(SubmittedVariantInactiveEntity::new) - .collect(Collectors.toList()) + ss1.getAccession(), "Hash mismatch with " + ss1.getClusteredVariantAccession(), + Stream.of(ss1, ss1_another).map(SubmittedVariantInactiveEntity::new) + .collect(Collectors.toList()) ); mongoTemplate.insert(Collections.singletonList(splitOperation), SubmittedVariantOperationEntity.class); @@ -381,15 +356,15 @@ public void testIdempotentSplitWrites() throws Exception { ss1 = createSS(1L, rs1Accession, 100L, "C", "T"); // Another SS with the same accession as SS1 but different REF/ALT SubmittedVariantEntity ss1_another = createSS(ss1.getAccession(), rs1Accession, 101L, - "T", ""); + "T", ""); SubmittedVariantOperationEntity splitOperation = createScenario2_SplitsInvolvingSameSSWithSameRS(ss1, ss1_another, rs1Accession); - rsSplitWriter.write(Collections.singletonList(splitOperation)); + rsSplitWriter.write(Chunk.of(splitOperation)); DatabaseState databaseStateAfterFirstSplitWrite = DatabaseState.getCurrentDatabaseState(this.mongoTemplate); - rsSplitWriter.write(Collections.singletonList(splitOperation)); + rsSplitWriter.write(Chunk.of(splitOperation)); DatabaseState databaseStateAfterSecondSplitWrite = DatabaseState.getCurrentDatabaseState(this.mongoTemplate); - rsSplitWriter.write(Collections.singletonList(splitOperation)); + rsSplitWriter.write(Chunk.of(splitOperation)); DatabaseState databaseStateAfterThirdSplitWrite = DatabaseState.getCurrentDatabaseState(this.mongoTemplate); assertEquals(databaseStateAfterSecondSplitWrite, databaseStateAfterFirstSplitWrite); @@ -425,7 +400,7 @@ public void testRetainedRSIsNotCreatedWhenHashExists() throws Exception { Stream.of(ss1, ss2).map(SubmittedVariantInactiveEntity::new).collect(Collectors.toList()) ); mongoTemplate.insert(Collections.singletonList(splitOperation), SubmittedVariantOperationEntity.class); - rsSplitWriter.write(Collections.singletonList(splitOperation)); + rsSplitWriter.write(Chunk.of(splitOperation)); // Ensure that SS1 gets to retain the old RS List> ssAssociatedWithRS1 = diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/SSSplitWriterTest.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/SSSplitWriterTest.java index dcba64060..cbe743db0 100644 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/SSSplitWriterTest.java +++ b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/SSSplitWriterTest.java @@ -15,26 +15,22 @@ */ package uk.ac.ebi.eva.accession.clustering.batch.io; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; -import com.mongodb.MongoClient; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mockito; import org.springframework.batch.core.JobExecution; +import org.springframework.batch.item.Chunk; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.test.mock.mockito.MockBean; -import org.springframework.context.ApplicationContext; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.http.HttpMethod; import org.springframework.http.HttpStatus; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import org.springframework.test.web.client.ExpectedCount; import org.springframework.test.web.client.MockRestServiceServer; import org.springframework.web.client.RestTemplate; @@ -44,7 +40,6 @@ import uk.ac.ebi.eva.accession.clustering.test.DatabaseState; import uk.ac.ebi.eva.accession.clustering.test.configuration.BatchTestConfiguration; import uk.ac.ebi.eva.accession.clustering.test.configuration.MongoTestConfiguration; -import uk.ac.ebi.eva.accession.clustering.test.rule.FixSpringMongoDbRule; import uk.ac.ebi.eva.accession.core.model.ISubmittedVariant; import uk.ac.ebi.eva.accession.core.model.SubmittedVariant; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpSubmittedVariantEntity; @@ -53,6 +48,7 @@ import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantOperationEntity; import uk.ac.ebi.eva.accession.core.service.nonhuman.SubmittedVariantAccessioningService; import uk.ac.ebi.eva.accession.core.summary.SubmittedVariantSummaryFunction; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.metrics.count.CountServiceParameters; import uk.ac.ebi.eva.metrics.metric.MetricCompute; @@ -64,9 +60,9 @@ import java.util.stream.Collectors; import java.util.stream.Stream; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.doCallRealMethod; import static org.mockito.Mockito.doThrow; import static org.springframework.test.web.client.match.MockRestRequestMatchers.method; @@ -74,19 +70,14 @@ import static org.springframework.test.web.client.response.MockRestResponseCreators.withStatus; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.SS_SPLIT_WRITER; -@RunWith(SpringRunner.class) +@ExtendWith(SpringExtension.class) @TestPropertySource("classpath:merge-split-test.properties") @ContextConfiguration(classes = {SSSplitWriterConfiguration.class, MongoTestConfiguration.class, BatchTestConfiguration.class}) -public class SSSplitWriterTest { - - private static final String TEST_DB = "test-db"; +public class SSSplitWriterTest extends MongoTestContainerHelper { private static final String ASSEMBLY = "GCA_000000001.1"; - @Autowired - private MongoClient mongoClient; - @Autowired private MongoTemplate mongoTemplate; @@ -94,14 +85,6 @@ public class SSSplitWriterTest { @Qualifier(SS_SPLIT_WRITER) private SSSplitWriter ssSplitWriter; - //Required by nosql-unit - @Autowired - private ApplicationContext applicationContext; - - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName(TEST_DB).build()); - private SubmittedVariantEntity ss1, ss2, ss3, ss4, ss5, ss6; private List svesThatShareSameID; @@ -126,26 +109,24 @@ public class SSSplitWriterTest { private final String URL_PATH_SAVE_COUNT = "/v1/bulk/count"; - @Before + @BeforeEach public void setUp() throws Exception { - cleanup(); + mongoTemplate.getDb().drop(); + metricCompute.clearCount(); + mockServer = MockRestServiceServer.createServer(restTemplate); mockServer.expect(ExpectedCount.manyTimes(), - requestTo(new URI(countServiceParameters.getUrl() + URL_PATH_SAVE_COUNT))) - .andExpect(method(HttpMethod.POST)) - .andRespond(withStatus(HttpStatus.OK)); + requestTo(new URI(countServiceParameters.getUrl() + URL_PATH_SAVE_COUNT))) + .andExpect(method(HttpMethod.POST)) + .andRespond(withStatus(HttpStatus.OK)); Mockito.when(jobExecution.getJobId()).thenReturn(1L); ssSplitWriter.setJobExecution(jobExecution); } - @After + @AfterEach public void tearDown() { - cleanup(); - } - - private void cleanup() { - mongoClient.dropDatabase(TEST_DB); + mongoTemplate.getDb().drop(); metricCompute.clearCount(); } @@ -154,7 +135,7 @@ private SubmittedVariantEntity createSS(Long ssAccession, Long rsAccession, Long Function hashingFunction = new SubmittedVariantSummaryFunction().andThen( new SHA1HashingFunction()); SubmittedVariant temp = new SubmittedVariant(ASSEMBLY, 60711, "PRJ1", "chr1", start, reference, alternate, - rsAccession); + rsAccession); temp.setSupportedByEvidence(supportedByEvidence); temp.setAllelesMatch(allelesMatch); return new SubmittedVariantEntity(ssAccession, hashingFunction.apply(temp), temp, 1); @@ -177,7 +158,7 @@ private void setupSplitScenario() { private void writeSplitWithoutCrashes() throws Exception { setupSplitScenario(); - ssSplitWriter.write(this.svesThatShareSameID); + ssSplitWriter.write(new Chunk<>(this.svesThatShareSameID)); } @Test @@ -198,22 +179,22 @@ private void assertPostSplitStatus() { DbsnpSubmittedVariantEntity.class); assertEquals(3, entriesInDbsnpSVE.size()); List expectedHashesInDbsnpSVE = Stream.of(ss4, ss5, ss6).map(SubmittedVariantEntity::getId) - .collect(Collectors.toList()); + .collect(Collectors.toList()); assertTrue(entriesInDbsnpSVE.stream().map(DbsnpSubmittedVariantEntity::getId) - .allMatch(expectedHashesInDbsnpSVE::contains)); + .allMatch(expectedHashesInDbsnpSVE::contains)); // other SS besides ss4 should be issued new IDs List entriesInSVE = this.mongoTemplate.findAll(SubmittedVariantEntity.class); assertEquals(3, entriesInSVE.size()); assertNotEquals(ss1.getAccession(), - this.submittedVariantAccessioningService.get(Collections.singletonList(ss1)).get(0) - .getAccession()); + this.submittedVariantAccessioningService.get(Collections.singletonList(ss1)).get(0) + .getAccession()); assertNotEquals(ss2.getAccession(), - this.submittedVariantAccessioningService.get(Collections.singletonList(ss2)).get(0) - .getAccession()); + this.submittedVariantAccessioningService.get(Collections.singletonList(ss2)).get(0) + .getAccession()); assertNotEquals(ss3.getAccession(), - this.submittedVariantAccessioningService.get(Collections.singletonList(ss3)).get(0) - .getAccession()); + this.submittedVariantAccessioningService.get(Collections.singletonList(ss3)).get(0) + .getAccession()); // check for creation of 3 new SS IDs assertEquals(3, this.metricCompute.getCount(ClusteringMetric.SUBMITTED_VARIANTS_SS_SPLIT)); @@ -232,14 +213,13 @@ public void testSplitWriterRestartAfterSplitCandidateRegistrationCrashes() throw doThrow(RuntimeException.class).when(mockSplitWriter).registerSplitCandidates(this.svesThatShareSameID); // invoke process - will crash when registering split candidates try { - mockSplitWriter.write(this.svesThatShareSameID); - } - catch (RuntimeException ignored) { + mockSplitWriter.write(new Chunk<>(this.svesThatShareSameID)); + } catch (RuntimeException ignored) { } doCallRealMethod().when(mockSplitWriter).registerSplitCandidates(svesThatShareSameID); // Restart process - mockSplitWriter.write(this.svesThatShareSameID); + mockSplitWriter.write(new Chunk<>(this.svesThatShareSameID)); assertPostSplitStatus(); } @@ -250,14 +230,13 @@ public void testSplitWriterRestartAfterSplitCandidateProcessingCrashes() throws doThrow(RuntimeException.class).when(mockSplitWriter).processSplitCandidates(Mockito.anyList()); // invoke process - will crash when processing split candidates try { - mockSplitWriter.write(this.svesThatShareSameID); - } - catch (RuntimeException ignored) { + mockSplitWriter.write(new Chunk<>(this.svesThatShareSameID)); + } catch (RuntimeException ignored) { } doCallRealMethod().when(mockSplitWriter).processSplitCandidates(Mockito.anyList()); // Restart process - mockSplitWriter.write(this.svesThatShareSameID); + mockSplitWriter.write(new Chunk<>(this.svesThatShareSameID)); assertPostSplitStatus(); } @@ -266,17 +245,16 @@ public void testSplitWriterRestartAfterSplitCandidateClearingCrashes() throws Ex setupSplitScenario(); SSSplitWriter mockSplitWriter = Mockito.spy((SSSplitWriter) ssSplitWriter); doThrow(RuntimeException.class).when(mockSplitWriter) - .removeCurrentSSEntriesInDBForSplitCandidates(Mockito.anySet()); + .removeCurrentSSEntriesInDBForSplitCandidates(Mockito.anySet()); // invoke process - will crash when clearing split candidates try { - mockSplitWriter.write(this.svesThatShareSameID); - } - catch (RuntimeException ignored) { + mockSplitWriter.write(new Chunk<>(this.svesThatShareSameID)); + } catch (RuntimeException ignored) { } doCallRealMethod().when(mockSplitWriter).removeCurrentSSEntriesInDBForSplitCandidates(Mockito.anySet()); // Restart process - mockSplitWriter.write(this.svesThatShareSameID); + mockSplitWriter.write(new Chunk<>(this.svesThatShareSameID)); assertPostSplitStatus(); } @@ -285,7 +263,7 @@ public void testIdempotentSplitWriter() throws Exception { writeSplitWithoutCrashes(); DatabaseState dbStateAfterFirstSplitWrite = DatabaseState.getCurrentDatabaseState(this.mongoTemplate); - ssSplitWriter.write(this.svesThatShareSameID); + ssSplitWriter.write(new Chunk<>(this.svesThatShareSameID)); DatabaseState dbStateAfterSecondSplitWrite = DatabaseState.getCurrentDatabaseState(this.mongoTemplate); assertEquals(dbStateAfterSecondSplitWrite, dbStateAfterFirstSplitWrite); } diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/StudyClusteringMongoReaderTest.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/StudyClusteringMongoReaderTest.java index bbbde7bd6..74c30f7f2 100644 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/StudyClusteringMongoReaderTest.java +++ b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/StudyClusteringMongoReaderTest.java @@ -15,27 +15,22 @@ */ package uk.ac.ebi.eva.accession.clustering.batch.io; -import com.lordofthejars.nosqlunit.annotation.UsingDataSet; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; -import com.mongodb.MongoClient; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.batch.item.ExecutionContext; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.ApplicationContext; +import org.springframework.core.io.ResourceLoader; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; - +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.eva.accession.clustering.test.configuration.MongoTestConfiguration; -import uk.ac.ebi.eva.accession.clustering.test.rule.FixSpringMongoDbRule; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.MongoConfiguration; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; +import uk.ac.ebi.eva.accession.core.utils.MongoTestDataLoader; import java.util.ArrayList; import java.util.Arrays; @@ -44,17 +39,12 @@ import java.util.stream.Collectors; import java.util.stream.Stream; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; -@RunWith(SpringRunner.class) +@ExtendWith(SpringExtension.class) @TestPropertySource("classpath:clustering-pipeline-test.properties") -@UsingDataSet(locations = {"/test-data/submittedVariantEntityStudyReader.json", - "/test-data/dbsnpSubmittedVariantEntityMongoReader.json"}) @ContextConfiguration(classes = {MongoConfiguration.class, MongoTestConfiguration.class}) -public class StudyClusteringMongoReaderTest { - - private static final String TEST_DB = "test-db"; - +public class StudyClusteringMongoReaderTest extends MongoTestContainerHelper { private static final String ASSEMBLY = "GCA_000000001.1"; private static final List PROJECTS = Arrays.asList("projectId_2", "projectId_3"); @@ -69,31 +59,29 @@ public class StudyClusteringMongoReaderTest { private StudyClusteringMongoReader studyClusteringMongoReader; - @Autowired - private MongoClient mongoClient; - @Autowired private MongoTemplate mongoTemplate; - //Required by nosql-unit @Autowired - private ApplicationContext applicationContext; + private ResourceLoader resourceLoader; + + @BeforeEach + public void setUp() { + mongoTemplate.getDb().drop(); - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName(TEST_DB).build()); + MongoTestDataLoader mongoTestDataLoader = new MongoTestDataLoader(mongoTemplate, resourceLoader); + mongoTestDataLoader.load("/test-data/submittedVariantEntityStudyReader.json"); + mongoTestDataLoader.load("/test-data/dbsnpSubmittedVariantEntityMongoReader.json"); - @Before - public void setUp(){ ExecutionContext executionContext = new ExecutionContext(); studyClusteringMongoReader = new StudyClusteringMongoReader(mongoTemplate, ASSEMBLY, PROJECTS, CHUNK_SIZE); studyClusteringMongoReader.open(executionContext); } - @After + @AfterEach public void tearDown() { studyClusteringMongoReader.close(); - mongoClient.dropDatabase(TEST_DB); + mongoTemplate.getDb().drop(); } @Test diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/clustering_writer/ClusteringAssertions.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/clustering_writer/ClusteringAssertions.java index 225d1bdb0..0d052cd98 100644 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/clustering_writer/ClusteringAssertions.java +++ b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/clustering_writer/ClusteringAssertions.java @@ -19,7 +19,7 @@ import uk.ac.ebi.eva.accession.clustering.metric.ClusteringMetric; import uk.ac.ebi.eva.metrics.metric.MetricCompute; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; public class ClusteringAssertions { diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/clustering_writer/IssueAccessionClusteringWriterTest.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/clustering_writer/IssueAccessionClusteringWriterTest.java index 14a8020cc..39f560d3b 100644 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/clustering_writer/IssueAccessionClusteringWriterTest.java +++ b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/clustering_writer/IssueAccessionClusteringWriterTest.java @@ -15,41 +15,37 @@ */ package uk.ac.ebi.eva.accession.clustering.batch.io.clustering_writer; -import com.lordofthejars.nosqlunit.annotation.UsingDataSet; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; import com.mongodb.client.FindIterable; import com.mongodb.client.MongoCollection; import org.bson.Document; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mockito; import org.springframework.batch.core.JobExecution; +import org.springframework.batch.item.Chunk; import org.springframework.batch.item.ExecutionContext; import org.springframework.batch.item.ItemWriter; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.boot.test.mock.mockito.MockBean; -import org.springframework.context.ApplicationContext; +import org.springframework.core.io.ResourceLoader; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.data.mongodb.core.query.Criteria; import org.springframework.data.mongodb.core.query.Query; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.ampt2d.commons.accession.hashing.SHA1HashingFunction; - import uk.ac.ebi.eva.accession.clustering.batch.io.ClusteringMongoReader; import uk.ac.ebi.eva.accession.clustering.batch.io.ClusteringWriter; import uk.ac.ebi.eva.accession.clustering.batch.io.RSSplitWriter; import uk.ac.ebi.eva.accession.clustering.parameters.InputParameters; import uk.ac.ebi.eva.accession.clustering.test.configuration.BatchTestConfiguration; -import uk.ac.ebi.eva.accession.clustering.test.rule.FixSpringMongoDbRule; +import uk.ac.ebi.eva.accession.clustering.test.configuration.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.ClusteredVariantAccessioningConfiguration; import uk.ac.ebi.eva.accession.core.model.ClusteredVariant; import uk.ac.ebi.eva.accession.core.model.IClusteredVariant; @@ -61,6 +57,8 @@ import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantOperationEntity; import uk.ac.ebi.eva.accession.core.summary.ClusteredVariantSummaryFunction; import uk.ac.ebi.eva.accession.core.summary.SubmittedVariantSummaryFunction; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; +import uk.ac.ebi.eva.accession.core.utils.MongoTestDataLoader; import uk.ac.ebi.eva.commons.core.models.VariantType; import uk.ac.ebi.eva.commons.mongodb.readers.MongoDbCursorItemReader; import uk.ac.ebi.eva.metrics.metric.MetricCompute; @@ -74,9 +72,9 @@ import java.util.List; import java.util.function.Function; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import static uk.ac.ebi.eva.accession.clustering.batch.io.clustering_writer.ClusteringAssertions.assertClusteringCounts; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.CLEAR_RS_MERGE_AND_SPLIT_CANDIDATES; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.CLUSTERED_CLUSTERING_WRITER; @@ -88,18 +86,16 @@ /** * This class handles the simplest scenarios of ClusteringWriter. - * + *

* The scenarios tested here are those about issuing new clustered variant accessions (RSs). No reusing existing RSs, * no RSs provided with the submitted variant being clustered, no RS merging. Other test classes in this folder take * care of those scenarios. */ -@RunWith(SpringRunner.class) +@ExtendWith(SpringExtension.class) @EnableAutoConfiguration -@ContextConfiguration(classes = {ClusteredVariantAccessioningConfiguration.class, BatchTestConfiguration.class}) +@ContextConfiguration(classes = {ClusteredVariantAccessioningConfiguration.class, BatchTestConfiguration.class, MongoTestConfiguration.class}) @TestPropertySource("classpath:clustering-issuance-test.properties") -public class IssueAccessionClusteringWriterTest { - - private static final String TEST_DB = "test-db"; +public class IssueAccessionClusteringWriterTest extends MongoTestContainerHelper { private static final String CLUSTERED_VARIANT_COLLECTION = "clusteredVariantEntity"; @@ -119,11 +115,10 @@ public class IssueAccessionClusteringWriterTest { private MongoTemplate mongoTemplate; @Autowired - private MetricCompute metricCompute; + private ResourceLoader resourceLoader; - //Required by nosql-unit @Autowired - private ApplicationContext applicationContext; + private MetricCompute metricCompute; // Current clustering sequence is: // generate merge split candidates from clustered variants -> perform merge @@ -163,12 +158,9 @@ public class IssueAccessionClusteringWriterTest { private Function clusteredHashingFunction; - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName(TEST_DB).build()); - - @Before + @BeforeEach public void setUp() throws IOException { + mongoTemplate.getDb().drop(); hashingFunction = new SubmittedVariantSummaryFunction().andThen(new SHA1HashingFunction()); clusteredHashingFunction = new ClusteredVariantSummaryFunction().andThen(new SHA1HashingFunction()); Files.deleteIfExists(this.rsReportFile.toPath()); @@ -179,16 +171,17 @@ public void setUp() throws IOException { clusteringWriterPreMergeAndSplit.setJobExecution(jobExecution); } - @After + @AfterEach public void tearDown() { mongoTemplate.getDb().drop(); metricCompute.clearCount(); } @Test - @UsingDataSet(locations = {"/test-data/submittedVariantEntity.json"}) @DirtiesContext public void writer() throws Exception { + new MongoTestDataLoader(mongoTemplate, resourceLoader).load("/test-data/submittedVariantEntity.json"); + List submittedVariantEntities = createSubmittedVariantEntities(); this.clusterVariants(submittedVariantEntities); assertClusteredVariantsCreated(); @@ -206,22 +199,22 @@ public void writer() throws Exception { private List createSubmittedVariantEntities() { List submittedVariantEntities = new ArrayList<>(); SubmittedVariant submittedVariant1 = createSubmittedVariant(inputParameters.getAssemblyAccession(), 1000, - PROJECT_ACCESSION, "1", 1000L, "T", "A"); + PROJECT_ACCESSION, "1", 1000L, "T", "A"); SubmittedVariantEntity submittedVariantEntity1 = createSubmittedVariantEntity(5000000001L, submittedVariant1); //Different alleles SubmittedVariant submittedVariant2 = createSubmittedVariant(inputParameters.getAssemblyAccession(), 1000, - PROJECT_ACCESSION, "1", 1000L, "T", "G"); + PROJECT_ACCESSION, "1", 1000L, "T", "G"); SubmittedVariantEntity submittedVariantEntity2 = createSubmittedVariantEntity(5000000002L, submittedVariant2); //Same assembly, contig, start but different type SubmittedVariant submittedVariantINS = createSubmittedVariant(inputParameters.getAssemblyAccession(), 1000, - PROJECT_ACCESSION, "1", 1000L, "", "A"); + PROJECT_ACCESSION, "1", 1000L, "", "A"); SubmittedVariantEntity submittedVariantEntityINS = createSubmittedVariantEntity(5000000003L, submittedVariantINS); SubmittedVariant submittedVariantDEL = createSubmittedVariant(inputParameters.getAssemblyAccession(), 1000, - PROJECT_ACCESSION, "1", 1000L, "T", ""); + PROJECT_ACCESSION, "1", 1000L, "T", ""); SubmittedVariantEntity submittedVariantEntityDEL = createSubmittedVariantEntity(5000000004L, submittedVariantDEL); //Different assembly, contig and start SubmittedVariant submittedVariant3 = createSubmittedVariant(inputParameters.getAssemblyAccession(), 3000, - PROJECT_ACCESSION, "1", 3000L, "C", "G"); + PROJECT_ACCESSION, "1", 3000L, "C", "G"); SubmittedVariantEntity submittedVariantEntity3 = createSubmittedVariantEntity(5000000005L, submittedVariant3); submittedVariantEntities.add(submittedVariantEntity1); submittedVariantEntities.add(submittedVariantEntity2); @@ -245,7 +238,7 @@ private SubmittedVariant createSubmittedVariant(String referenceSequenceAccessio String projectAccession, String contig, long start, String referenceAllele, String alternateAllele) { return new SubmittedVariant(referenceSequenceAccession, taxonomyAccession, projectAccession, contig, start, - referenceAllele, alternateAllele, null); + referenceAllele, alternateAllele, null); } private void assertClusteredVariantsCreated() { @@ -278,14 +271,14 @@ private boolean allSubmittedVariantsClustered() { MongoCollection collection = mongoTemplate.getCollection(SUBMITTED_VARIANT_COLLECTION); FindIterable documents = collection.find(); for (Document document : documents) { - if (document.get("rs") == null){ + if (document.get("rs") == null) { return false; } } return true; } - private void assertSubmittedVariantsOperationInserted(){ + private void assertSubmittedVariantsOperationInserted() { MongoCollection collection = mongoTemplate.getCollection(SUBMITTED_VARIANT_OPERATION_COLLECTION); assertEquals(5, collection.countDocuments()); List expectedAccessions = Arrays.asList(5000000001L, 5000000002L, 5000000003L, 5000000004L, 5000000005L); @@ -300,11 +293,11 @@ public void cluster_eva_submitted_variant_into_a_dbsnp_not_multimap_clustered_va // given Long rsAccession = 30L; ClusteredVariantEntity rs1 = createClusteredVariantEntity(rsAccession, inputParameters.getAssemblyAccession(), - "chr1", null); + "chr1", null); SubmittedVariantEntity ssToCluster = createSubmittedVariantEntity(5200000000L, - inputParameters.getAssemblyAccession(), - "chr1", "project2", null); + inputParameters.getAssemblyAccession(), + "chr1", "project2", null); mongoTemplate.insert(Arrays.asList(rs1), DbsnpClusteredVariantEntity.class); mongoTemplate.insert(Arrays.asList(ssToCluster), SubmittedVariantEntity.class); @@ -328,27 +321,27 @@ public void do_not_cluster_eva_submitted_variant_into_a_dbsnp_multimap_clustered throws Exception { // given ClusteredVariantEntity rs1Locus1 = createClusteredVariantEntity(30L, inputParameters.getAssemblyAccession(), - "chr1", 3); + "chr1", 3); ClusteredVariantEntity rs1Locus2 = createClusteredVariantEntity(30L, inputParameters.getAssemblyAccession(), - "chr2", null); + "chr2", null); SubmittedVariantEntity ssToCluster = createSubmittedVariantEntity(5200000000L, - inputParameters.getAssemblyAccession(), - "chr1", "project2", - null); + inputParameters.getAssemblyAccession(), + "chr1", "project2", + null); mongoTemplate.insert(Arrays.asList(rs1Locus1, rs1Locus2), DbsnpClusteredVariantEntity.class); mongoTemplate.insert(Arrays.asList(ssToCluster), SubmittedVariantEntity.class); Query querySsToCluster = new Query(new Criteria("accession").is(ssToCluster.getAccession())); assertNull(mongoTemplate.find(querySsToCluster, SubmittedVariantEntity.class).get(0) - .getClusteredVariantAccession()); + .getClusteredVariantAccession()); // when this.clusterVariants(Collections.singletonList(ssToCluster)); // then assertNull(mongoTemplate.find(querySsToCluster, SubmittedVariantEntity.class).get(0) - .getClusteredVariantAccession()); + .getClusteredVariantAccession()); assertClusteringCounts(metricCompute, 0, 0, 0, 1, 0, 0, 0); } @@ -357,11 +350,11 @@ public void do_not_cluster_eva_submitted_variant_into_a_dbsnp_multimap_clustered public void do_not_cluster_eva_submitted_variant_into_a_dbsnp_multimap_clustered_variant() throws Exception { // given ClusteredVariantEntity rs1 = createClusteredVariantEntity(30L, inputParameters.getAssemblyAccession(), "chr1", - 3); + 3); SubmittedVariantEntity ssToCluster = createSubmittedVariantEntity(5200000000L, - inputParameters.getAssemblyAccession(), - "chr1", "project2", - null); + inputParameters.getAssemblyAccession(), + "chr1", "project2", + null); mongoTemplate.insert(Arrays.asList(rs1), DbsnpClusteredVariantEntity.class); mongoTemplate.insert(Arrays.asList(ssToCluster), SubmittedVariantEntity.class); @@ -388,8 +381,8 @@ private ClusteredVariantEntity createClusteredVariantEntity(Long accession, Stri ClusteredVariant variant = createClusteredVariant(assemblyAccession, contig); String hash = clusteredHashingFunction.apply(variant); ClusteredVariantEntity variantEntity = new ClusteredVariantEntity(accession, hash, assemblyAccession, 1000, - contig, 100, VariantType.SNV, false, null, 1, - mapWeight); + contig, 100, VariantType.SNV, false, null, 1, + mapWeight); return variantEntity; } @@ -405,38 +398,38 @@ private SubmittedVariantEntity createSubmittedVariantEntity(Long accession, Stri private void clusterVariants(List submittedVariantEntities) throws Exception { - clusteringWriterPreMergeAndSplit.write(submittedVariantEntities); + clusteringWriterPreMergeAndSplit.write(new Chunk<>(submittedVariantEntities)); List mergeCandidates = new ArrayList<>(); List splitCandidates = new ArrayList<>(); SubmittedVariantOperationEntity tempSVO; rsMergeCandidatesReader.open(new ExecutionContext()); - while((tempSVO = rsMergeCandidatesReader.read()) != null) { + while ((tempSVO = rsMergeCandidatesReader.read()) != null) { mergeCandidates.add(tempSVO); } rsSplitCandidatesReader.open(new ExecutionContext()); - while((tempSVO = rsSplitCandidatesReader.read()) != null) { + while ((tempSVO = rsSplitCandidatesReader.read()) != null) { splitCandidates.add(tempSVO); } - rsMergeWriter.write(mergeCandidates); - rsSplitWriter.write(splitCandidates); + rsMergeWriter.write(new Chunk<>(mergeCandidates)); + rsSplitWriter.write(new Chunk<>(splitCandidates)); ClusteringMongoReader unclusteredVariantsReader = new ClusteringMongoReader(this.mongoTemplate, - inputParameters.getAssemblyAccession(), - 100, - false); + inputParameters.getAssemblyAccession(), + 100, + false); unclusteredVariantsReader.initializeReader(); List unclusteredVariants = new ArrayList<>(); SubmittedVariantEntity tempSV; - while((tempSV = unclusteredVariantsReader.read()) != null) { + while ((tempSV = unclusteredVariantsReader.read()) != null) { unclusteredVariants.add(tempSV); } unclusteredVariantsReader.close(); // Cluster non-clustered variants - clusteringWriterPostMergeAndSplit.write(unclusteredVariants); + clusteringWriterPostMergeAndSplit.write(new Chunk<>(unclusteredVariants)); // Spring has a mandatory requirement of even small functionality being writers. // To satisfy that, we pass in a dummy object to invoke the writer // which basically clears the merge and split operations after they were processed above - clearRSMergeAndSplitCandidates.write(Collections.singletonList(new Object())); + clearRSMergeAndSplitCandidates.write(Chunk.of(new Object())); } } diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/clustering_writer/MergeAccessionClusteringWriterTest.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/clustering_writer/MergeAccessionClusteringWriterTest.java index b0ef3d4e2..dd014267f 100644 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/clustering_writer/MergeAccessionClusteringWriterTest.java +++ b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/clustering_writer/MergeAccessionClusteringWriterTest.java @@ -15,35 +15,30 @@ */ package uk.ac.ebi.eva.accession.clustering.batch.io.clustering_writer; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; import org.assertj.core.util.Sets; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.batch.item.Chunk; import org.springframework.batch.item.ExecutionContext; import org.springframework.batch.item.ItemWriter; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; -import org.springframework.context.ApplicationContext; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.data.mongodb.core.query.Query; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; - +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.ampt2d.commons.accession.core.models.EventType; import uk.ac.ebi.ampt2d.commons.accession.hashing.SHA1HashingFunction; import uk.ac.ebi.ampt2d.commons.accession.persistence.mongodb.document.EventDocument; - import uk.ac.ebi.eva.accession.clustering.batch.io.ClusteringMongoReader; import uk.ac.ebi.eva.accession.clustering.batch.io.ClusteringWriter; import uk.ac.ebi.eva.accession.clustering.test.configuration.BatchTestConfiguration; -import uk.ac.ebi.eva.accession.clustering.test.rule.FixSpringMongoDbRule; +import uk.ac.ebi.eva.accession.clustering.test.configuration.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.ClusteredVariantAccessioningConfiguration; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.SubmittedVariantAccessioningConfiguration; import uk.ac.ebi.eva.accession.core.model.ClusteredVariant; @@ -62,6 +57,7 @@ import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantOperationEntity; import uk.ac.ebi.eva.accession.core.summary.ClusteredVariantSummaryFunction; import uk.ac.ebi.eva.accession.core.summary.SubmittedVariantSummaryFunction; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.commons.core.models.VariantType; import uk.ac.ebi.eva.commons.mongodb.readers.MongoDbCursorItemReader; import uk.ac.ebi.eva.metrics.metric.MetricCompute; @@ -74,9 +70,9 @@ import java.util.function.Function; import java.util.stream.Collectors; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import static uk.ac.ebi.eva.accession.clustering.batch.io.clustering_writer.ClusteringAssertions.assertClusteringCounts; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.CLEAR_RS_MERGE_AND_SPLIT_CANDIDATES; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.CLUSTERED_CLUSTERING_WRITER; @@ -92,20 +88,17 @@ /** * This class handles some scenarios of ClusteringWriter where redundant RSs are discovered and merged. - * + *

* That includes all the scenarios that involve writing to the Operation collections, e.g. * clusteredVariantOperationEntity to register an RS merge. * Other test classes in this folder take care of other scenarios. */ -@RunWith(SpringRunner.class) +@ExtendWith(SpringExtension.class) @EnableAutoConfiguration @ContextConfiguration(classes = {ClusteredVariantAccessioningConfiguration.class, - SubmittedVariantAccessioningConfiguration.class, BatchTestConfiguration.class}) + SubmittedVariantAccessioningConfiguration.class, BatchTestConfiguration.class, MongoTestConfiguration.class}) @TestPropertySource("classpath:clustering-writer-test.properties") -public class MergeAccessionClusteringWriterTest { - - private static final String TEST_DB = "test-db"; - +public class MergeAccessionClusteringWriterTest extends MongoTestContainerHelper { private static final String CLUSTERED_VARIANT_COLLECTION = "clusteredVariantEntity"; private static final String SUBMITTED_VARIANT_COLLECTION = "submittedVariantEntity"; @@ -166,30 +159,23 @@ public class MergeAccessionClusteringWriterTest { @Autowired private Long accessioningMonotonicInitSs; - //Required by nosql-unit - @Autowired - private ApplicationContext applicationContext; - private Function hashingFunction; private Function clusteredHashingFunction; - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName(TEST_DB).build()); - private static final String defaultReferenceAlleleForTesting = "T"; private static final String defaultAlternateAlleleForTesting = "A"; - @Before + @BeforeEach public void setUp() { mongoTemplate.getDb().drop(); + metricCompute.clearCount(); hashingFunction = new SubmittedVariantSummaryFunction().andThen(new SHA1HashingFunction()); clusteredHashingFunction = new ClusteredVariantSummaryFunction().andThen(new SHA1HashingFunction()); } - @After + @AfterEach public void tearDown() { mongoTemplate.getDb().drop(); metricCompute.clearCount(); @@ -287,20 +273,20 @@ public void mergeClusteredAccession(Long rs1, Long rs2, Long ssToRemap, Long ss2 // ssToRemap in the old assembly, will be remapped to asm2 (see sve1Remapped below) mongoTemplate.insert(createSubmittedVariantEntity(ASM_1, START, rs1, ssToRemap, NOT_REMAPPED), - getSubmittedCollection(ssToRemap)); + getSubmittedCollection(ssToRemap)); // ss2 in the new assembly, will change its RS to rs1 when we realise rs1 and rs2 should be merged // because they both map to the same position in asm2 mongoTemplate.insert(createSubmittedVariantEntity(ASM_2, START, rs2, ss2, NOT_REMAPPED), getSubmittedCollection(ss2)); assertDatabaseCounts(expectedDbsnpCve - additionalDbsnpCveCreatedPostMerge, - expectedCve - additionalCveCreatedPostMerge, 0, 0, - expectedDbsnpSve, expectedSve, 0, 0); + expectedCve - additionalCveCreatedPostMerge, 0, 0, + expectedDbsnpSve, expectedSve, 0, 0); // when SubmittedVariantEntity sve1Remapped = createSubmittedVariantEntity(ASM_2, START, rs1, ssToRemap, ASM_1, - defaultAlternateAlleleForTesting, - defaultReferenceAlleleForTesting); + defaultAlternateAlleleForTesting, + defaultReferenceAlleleForTesting); // To ensure that we don't create collision with ss2, // use alleles other than defaultAlternateAlleleForTesting or defaultReferenceAlleleForTesting mongoTemplate.insert(sve1Remapped, getSubmittedCollection(sve1Remapped.getAccession())); @@ -314,7 +300,7 @@ public void mergeClusteredAccession(Long rs1, Long rs2, Long ssToRemap, Long ss2 // then assertDatabaseCounts(expectedDbsnpCve, expectedCve, expectedDbsnpCvOperations, expectedCvOperations, - expectedDbsnpSve, expectedSve, expectedDbsnpSvOperations, expectedSvOperations); + expectedDbsnpSve, expectedSve, expectedDbsnpSvOperations, expectedSvOperations); assertAssembliesPresent(Sets.newTreeSet(ASM_1, ASM_2)); } @@ -325,12 +311,11 @@ public void mergeClusteredAccession(Long rs1, Long rs2, Long ssToRemap, Long ss2 int expectedDbsnpSve, int expectedSve, int expectedDbsnpSvOperations, int expectedSvOperations) throws Exception { mergeClusteredAccession(rs1, rs2, ssToRemap, ss2, expectedDbsnpCve, expectedCve, expectedDbsnpCvOperations, - expectedCvOperations, expectedDbsnpSve, expectedSve, expectedDbsnpSvOperations, - expectedSvOperations, 0, 0); + expectedCvOperations, expectedDbsnpSve, expectedSve, expectedDbsnpSvOperations, + expectedSvOperations, 0, 0); } - private ClusteredVariantEntity createClusteredVariantEntity(String assembly, Long rs) { ClusteredVariant cv = new ClusteredVariant(assembly, 1000, "1", 100L, VariantType.SNV, false, null); String cvHash = clusteredHashingFunction.apply(cv); @@ -360,12 +345,12 @@ private void assertDatabaseCounts(int expectedDbsnpCve, int expectedCve, assertEquals(expectedDbsnpCve, mongoTemplate.count(new Query(), DbsnpClusteredVariantEntity.class)); assertEquals(expectedCve, mongoTemplate.count(new Query(), ClusteredVariantEntity.class)); assertEquals(expectedDbsnpCvOperations, mongoTemplate.count(new Query(), - DbsnpClusteredVariantOperationEntity.class)); + DbsnpClusteredVariantOperationEntity.class)); assertEquals(expectedCvOperations, mongoTemplate.count(new Query(), ClusteredVariantOperationEntity.class)); assertEquals(expectedDbsnpSve, mongoTemplate.count(new Query(), DbsnpSubmittedVariantEntity.class)); assertEquals(expectedSve, mongoTemplate.count(new Query(), SubmittedVariantEntity.class)); assertEquals(expectedDbsnpSvOperations, mongoTemplate.count(new Query(), - DbsnpSubmittedVariantOperationEntity.class)); + DbsnpSubmittedVariantOperationEntity.class)); assertEquals(expectedSvOperations, mongoTemplate.count(new Query(), SubmittedVariantOperationEntity.class)); } @@ -373,14 +358,14 @@ private void assertMergedInto(Long mergedInto, Long originalAccession, Long... u // Confine the variants being inspected to the remapped assembly List submittedVariants = mongoTemplate - .findAll(SubmittedVariantEntity.class) - .stream().filter(entity -> entity.getReferenceSequenceAccession().equals(ASM_2)) - .collect(Collectors.toList()); + .findAll(SubmittedVariantEntity.class) + .stream().filter(entity -> entity.getReferenceSequenceAccession().equals(ASM_2)) + .collect(Collectors.toList()); submittedVariants.addAll(mongoTemplate - .findAll(DbsnpSubmittedVariantEntity.class) - .stream() - .filter(entity -> entity.getReferenceSequenceAccession().equals(ASM_2)) - .collect(Collectors.toList())); + .findAll(DbsnpSubmittedVariantEntity.class) + .stream() + .filter(entity -> entity.getReferenceSequenceAccession().equals(ASM_2)) + .collect(Collectors.toList())); assertClusteredVariantAccessionEqual(Sets.newTreeSet(mergedInto), submittedVariants); // Confine the variants being inspected to the remapped assembly @@ -390,9 +375,9 @@ private void assertMergedInto(Long mergedInto, Long originalAccession, Long... u .stream().filter(entity -> entity.getAssemblyAccession().equals(ASM_2)) .collect(Collectors.toList()); clusteredVariants.addAll(mongoTemplate - .findAll(DbsnpClusteredVariantEntity.class) - .stream().filter(entity -> entity.getAssemblyAccession().equals(ASM_2)) - .collect(Collectors.toList())); + .findAll(DbsnpClusteredVariantEntity.class) + .stream().filter(entity -> entity.getAssemblyAccession().equals(ASM_2)) + .collect(Collectors.toList())); assertAccessionEqual(Sets.newTreeSet(mergedInto), clusteredVariants); EventDocument clusteredOp = @@ -412,8 +397,8 @@ private void assertMergedInto(Long mergedInto, Long originalAccession, Long... u } assertTrue(submittedOps.stream().map(EventDocument::getEventType).allMatch(EventType.UPDATED::equals)); assertEquals(Sets.newTreeSet(updatedSubmittedVariant), submittedOps.stream() - .map(EventDocument::getAccession) - .collect(Collectors.toSet())); + .map(EventDocument::getAccession) + .collect(Collectors.toSet())); } @Test @@ -433,7 +418,7 @@ public void do_not_merge_into_remapped_multimap_variants() throws Exception { mongoTemplate.insert(createClusteredVariantEntity(asm1, 200L, rs1, null), getClusteredTable(rs1)); mongoTemplate.insert(createSubmittedVariantEntity(asm1, 100L, rs1, ssToRemap, NOT_REMAPPED), - getSubmittedCollection(ssToRemap)); + getSubmittedCollection(ssToRemap)); mongoTemplate.insert(createSubmittedVariantEntity(asm1, 200L, rs1, ss3, NOT_REMAPPED), getSubmittedCollection(ssToRemap)); @@ -443,7 +428,7 @@ public void do_not_merge_into_remapped_multimap_variants() throws Exception { mongoTemplate.insert(createSubmittedVariantEntity(asm2, 100L, rs2, ss2, NOT_REMAPPED), getSubmittedCollection(ss2)); assertDatabaseCounts(0, 3, 0, 0, - 0, 3, 0, 0); + 0, 3, 0, 0); // when SubmittedVariantEntity sve1Remapped = createSubmittedVariantEntity(asm2, 100L, rs1, ssToRemap, asm1); @@ -452,7 +437,7 @@ public void do_not_merge_into_remapped_multimap_variants() throws Exception { // then assertDatabaseCounts(0, 3, 0, 0, - 0, 3, 0, 0); + 0, 3, 0, 0); assertAssembliesPresent(Sets.newTreeSet(asm1, asm2)); } @@ -475,7 +460,7 @@ public void do_not_merge_remapped_multimap_variants() throws Exception { mongoTemplate.insert(createClusteredVariantEntity(asm1, 200L, rs1, 3), getClusteredTable(rs1)); mongoTemplate.insert(createSubmittedVariantEntity(asm1, 100L, rs1, ssToRemap, NOT_REMAPPED), - getSubmittedCollection(ssToRemap)); + getSubmittedCollection(ssToRemap)); mongoTemplate.insert(createSubmittedVariantEntity(asm1, 200L, rs1, ss3, NOT_REMAPPED), getSubmittedCollection(ssToRemap)); @@ -484,7 +469,7 @@ public void do_not_merge_remapped_multimap_variants() throws Exception { mongoTemplate.insert(createSubmittedVariantEntity(asm2, 100L, rs2, ss2, NOT_REMAPPED), getSubmittedCollection(ss2)); assertDatabaseCounts(0, 3, 0, 0, - 0, 3, 0, 0); + 0, 3, 0, 0); // when SubmittedVariantEntity sve1Remapped = createSubmittedVariantEntity(asm2, 100L, rs1, ssToRemap, asm1); @@ -493,7 +478,7 @@ public void do_not_merge_remapped_multimap_variants() throws Exception { // then assertDatabaseCounts(0, 3, 0, 0, - 0, 3, 0, 0); + 0, 3, 0, 0); assertAssembliesPresent(Sets.newTreeSet(asm1, asm2)); } @@ -526,7 +511,7 @@ public void merge_into_remapped_multimap_variants_if_single_mapping_per_assembly mongoTemplate.insert(createSubmittedVariantEntity(asm2, 200L, rs2, ss2, NOT_REMAPPED), getSubmittedCollection(ss2)); assertDatabaseCounts(0, 3, 0, 0, - 0, 3, 0, 0); + 0, 3, 0, 0); // when SubmittedVariantEntity sve1Remapped = createSubmittedVariantEntity(asm2, 200L, rs1, ssToRemap, asm1); @@ -534,7 +519,7 @@ public void merge_into_remapped_multimap_variants_if_single_mapping_per_assembly // then assertDatabaseCounts(0, 3, 0, 1, - 0, 3, 0, 1); + 0, 3, 0, 1); assertAssembliesPresent(Sets.newTreeSet(asm1, asm2, asm3)); assertMergedInto(rs1, rs2, ss2); @@ -568,7 +553,7 @@ public void merge_multimap_variants_into_remapped_variants_if_single_mapping_per mongoTemplate.insert(createSubmittedVariantEntity(asm3, 300L, rs2, ss3, NOT_REMAPPED), getSubmittedCollection(ss3)); assertDatabaseCounts(0, 3, 0, 0, - 0, 3, 0, 0); + 0, 3, 0, 0); // when SubmittedVariantEntity sve1Remapped = createSubmittedVariantEntity(asm2, 200L, rs1, ssToRemap, asm1); @@ -577,7 +562,7 @@ public void merge_multimap_variants_into_remapped_variants_if_single_mapping_per // then // RS2 to RS1 merge will be confined to the ASM2 assembly. Therefore number of CV and SV operations is just 1. assertDatabaseCounts(0, 3, 0, 1, - 0, 3, 0, 1); + 0, 3, 0, 1); assertAssembliesPresent(Sets.newTreeSet(asm1, asm2, asm3)); assertMergedInto(rs1, rs2, ss2); @@ -598,7 +583,7 @@ private ClusteredVariantEntity createClusteredVariantEntity(String assembly, Lon ClusteredVariant cv = new ClusteredVariant(assembly, 1000, "1", start, VariantType.SNV, false, null); String cvHash = clusteredHashingFunction.apply(cv); return new ClusteredVariantEntity(rs, cvHash, assembly, 1000, "1", start, VariantType.SNV, false, null, 1, - mapWeight); + mapWeight); } private SubmittedVariantEntity createSubmittedVariantEntity(String assembly, Long start, Long rs, Long ss, @@ -607,8 +592,8 @@ private SubmittedVariantEntity createSubmittedVariantEntity(String assembly, Lon SubmittedVariant submittedClustered = new SubmittedVariant(assembly, 1000, "project", "1", start, reference, alternate, rs); String hash1 = hashingFunction.apply(submittedClustered); SubmittedVariantEntity submittedVariantEntity = new SubmittedVariantEntity(ss, hash1, submittedClustered, 1, - remappedFrom, LocalDateTime.now(), - null); + remappedFrom, LocalDateTime.now(), + null); submittedVariantEntity.setMapWeight(mapWeight); return submittedVariantEntity; } @@ -616,53 +601,53 @@ private SubmittedVariantEntity createSubmittedVariantEntity(String assembly, Lon private SubmittedVariantEntity createSubmittedVariantEntity(String assembly, Long start, Long rs, Long ss, String remappedFrom, Integer mapWeight) { return createSubmittedVariantEntity(assembly, start, rs, ss, remappedFrom, mapWeight, - defaultReferenceAlleleForTesting, defaultAlternateAlleleForTesting); + defaultReferenceAlleleForTesting, defaultAlternateAlleleForTesting); } private SubmittedVariantEntity createSubmittedVariantEntity(String assembly, Long start, Long rs, Long ss, String remappedFrom) { return createSubmittedVariantEntity(assembly, start, rs, ss, remappedFrom, null, - defaultReferenceAlleleForTesting, defaultAlternateAlleleForTesting); + defaultReferenceAlleleForTesting, defaultAlternateAlleleForTesting); } private SubmittedVariantEntity createSubmittedVariantEntity(String assembly, Long start, Long rs, Long ss, String remappedFrom, String reference, String alternate) { return createSubmittedVariantEntity(assembly, start, rs, ss, remappedFrom, null, - reference, alternate); + reference, alternate); } private void clusterVariants(List submittedVariantEntities) throws Exception { - clusteringWriterPreMergeAndSplit.write(submittedVariantEntities); + clusteringWriterPreMergeAndSplit.write(new Chunk<>(submittedVariantEntities)); List mergeCandidates = new ArrayList<>(); List splitCandidates = new ArrayList<>(); SubmittedVariantOperationEntity tempSVO; rsMergeCandidatesReader.open(new ExecutionContext()); - while((tempSVO = rsMergeCandidatesReader.read()) != null) { + while ((tempSVO = rsMergeCandidatesReader.read()) != null) { mergeCandidates.add(tempSVO); } rsSplitCandidatesReader.open(new ExecutionContext()); - while((tempSVO = rsSplitCandidatesReader.read()) != null) { + while ((tempSVO = rsSplitCandidatesReader.read()) != null) { splitCandidates.add(tempSVO); } - rsMergeWriter.write(mergeCandidates); - rsSplitWriter.write(splitCandidates); + rsMergeWriter.write(new Chunk<>(mergeCandidates)); + rsSplitWriter.write(new Chunk<>(splitCandidates)); ClusteringMongoReader unclusteredVariantsReader = new ClusteringMongoReader(this.mongoTemplate, ASM_2, 100, - false); + false); unclusteredVariantsReader.initializeReader(); List unclusteredVariants = new ArrayList<>(); SubmittedVariantEntity tempSV; - while((tempSV = unclusteredVariantsReader.read()) != null) { + while ((tempSV = unclusteredVariantsReader.read()) != null) { unclusteredVariants.add(tempSV); } unclusteredVariantsReader.close(); // Cluster non-clustered variants - clusteringWriterPostMergeAndSplit.write(unclusteredVariants); + clusteringWriterPostMergeAndSplit.write(new Chunk<>(unclusteredVariants)); // Spring has a mandatory requirement of even small functionality being writers. // To satisfy that, we pass in a dummy object to invoke the writer // which basically clears the merge and split operations after they were processed above - clearRSMergeAndSplitCandidates.write(Collections.singletonList(new Object())); + clearRSMergeAndSplitCandidates.write(Chunk.of(new Object())); } } diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/clustering_writer/RemappedVariantsClusteringWriterTest.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/clustering_writer/RemappedVariantsClusteringWriterTest.java index 05f1b2f27..95be884ea 100644 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/clustering_writer/RemappedVariantsClusteringWriterTest.java +++ b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/clustering_writer/RemappedVariantsClusteringWriterTest.java @@ -15,36 +15,32 @@ */ package uk.ac.ebi.eva.accession.clustering.batch.io.clustering_writer; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mockito; import org.springframework.batch.core.JobExecution; +import org.springframework.batch.item.Chunk; import org.springframework.batch.item.ExecutionContext; import org.springframework.batch.item.ItemWriter; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.boot.test.mock.mockito.MockBean; -import org.springframework.context.ApplicationContext; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.ampt2d.commons.accession.core.models.EventType; import uk.ac.ebi.ampt2d.commons.accession.hashing.SHA1HashingFunction; - import uk.ac.ebi.eva.accession.clustering.batch.io.ClusteringMongoReader; import uk.ac.ebi.eva.accession.clustering.batch.io.ClusteringWriter; import uk.ac.ebi.eva.accession.clustering.batch.io.RSSplitWriter; import uk.ac.ebi.eva.accession.clustering.parameters.InputParameters; import uk.ac.ebi.eva.accession.clustering.test.configuration.BatchTestConfiguration; -import uk.ac.ebi.eva.accession.clustering.test.rule.FixSpringMongoDbRule; +import uk.ac.ebi.eva.accession.clustering.test.configuration.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.ClusteredVariantAccessioningConfiguration; import uk.ac.ebi.eva.accession.core.model.ClusteredVariant; import uk.ac.ebi.eva.accession.core.model.IClusteredVariant; @@ -57,6 +53,7 @@ import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantOperationEntity; import uk.ac.ebi.eva.accession.core.summary.ClusteredVariantSummaryFunction; import uk.ac.ebi.eva.accession.core.summary.SubmittedVariantSummaryFunction; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.commons.core.models.VariantType; import uk.ac.ebi.eva.commons.mongodb.readers.MongoDbCursorItemReader; @@ -70,8 +67,8 @@ import java.util.function.Function; import java.util.stream.Collectors; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.BACK_PROPAGATED_RS_WRITER; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.CLEAR_RS_MERGE_AND_SPLIT_CANDIDATES; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.CLUSTERED_CLUSTERING_WRITER; @@ -86,14 +83,12 @@ *

* Other test classes in this folder take care of other scenarios. */ -@RunWith(SpringRunner.class) +@ExtendWith(SpringExtension.class) @EnableAutoConfiguration -@ContextConfiguration(classes = {ClusteredVariantAccessioningConfiguration.class, BatchTestConfiguration.class}) +@ContextConfiguration(classes = {ClusteredVariantAccessioningConfiguration.class, BatchTestConfiguration.class, + MongoTestConfiguration.class}) @TestPropertySource("classpath:clustering-writer-test.properties") -public class RemappedVariantsClusteringWriterTest { - - private static final String TEST_DB = "test-db"; - +public class RemappedVariantsClusteringWriterTest extends MongoTestContainerHelper { private static final String CLUSTERED_VARIANT_COLLECTION = "clusteredVariantEntity"; private static final String SUBMITTED_VARIANT_COLLECTION = "submittedVariantEntity"; @@ -160,15 +155,7 @@ public class RemappedVariantsClusteringWriterTest { private Function clusteredHashingFunction; - //Required by nosql-unit - @Autowired - private ApplicationContext applicationContext; - - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName(TEST_DB).build()); - - @Before + @BeforeEach public void setUp() { ASM_1 = inputParameters.getRemappedFrom(); ASM_2 = inputParameters.getAssemblyAccession(); @@ -182,7 +169,7 @@ public void setUp() { clusteringWriterPostMergeAndSplit.setJobExecution(jobExecution); } - @After + @AfterEach public void tearDown() { mongoTemplate.getDb().drop(); } @@ -191,7 +178,7 @@ public void tearDown() { @DirtiesContext public void test_variant_clustered_rs_id_does_not_exist() throws Exception { SubmittedVariantEntity submittedVariantEntity1 = getSubmittedVariantEntity(ASM_1, PROJECT_1, 1000, 3000000001L, - 5000000000L, NOT_REMAPPED); + 5000000000L, NOT_REMAPPED); mongoTemplate.insert(submittedVariantEntity1, SUBMITTED_VARIANT_COLLECTION); ClusteredVariantEntity clusteredVariantEntity1 = getClusteredVariantEntity(ASM_1, 1000, 3000000001L); @@ -199,7 +186,7 @@ public void test_variant_clustered_rs_id_does_not_exist() throws Exception { //variant remapped from asm1 to asm2 SubmittedVariantEntity submittedVariantEntity2 = getSubmittedVariantEntity(ASM_2, PROJECT_1, 2000, 3000000001L, - 5000000000L, ASM_1); + 5000000000L, ASM_1); mongoTemplate.insert(submittedVariantEntity2, SUBMITTED_VARIANT_COLLECTION); //asm2 clustered @@ -209,9 +196,9 @@ public void test_variant_clustered_rs_id_does_not_exist() throws Exception { //get all submitted variants with assembly asm2 and assert rs id List submittedVariants = mongoTemplate.findAll(SubmittedVariantEntity.class).stream() - .filter(s -> s.getReferenceSequenceAccession() - .equals(ASM_2)) - .collect(Collectors.toList()); + .filter(s -> s.getReferenceSequenceAccession() + .equals(ASM_2)) + .collect(Collectors.toList()); assertEquals(1, submittedVariants.size()); assertEquals(3000000001L, submittedVariants.get(0).getClusteredVariantAccession().longValue()); @@ -221,9 +208,9 @@ public void test_variant_clustered_rs_id_does_not_exist() throws Exception { assertEquals(3000000001L, clusteredVariants.get(0).getAccession().longValue()); assertEquals(3000000001L, clusteredVariants.get(1).getAccession().longValue()); assertEquals(ASM_1, clusteredVariants.stream().filter(s -> s.getStart() == 1000).collect(Collectors.toList()) - .get(0).getAssemblyAccession()); + .get(0).getAssemblyAccession()); assertEquals(ASM_2, clusteredVariants.stream().filter(s -> s.getStart() == 2000).collect(Collectors.toList()) - .get(0).getAssemblyAccession()); + .get(0).getAssemblyAccession()); //assert no submitted variant operations should happen List submittedVariantOperations = mongoTemplate.findAll( @@ -240,10 +227,10 @@ public void test_variant_clustered_rs_id_does_not_exist() throws Exception { @DirtiesContext public void test_variant_clustered_rs_id_exist() throws Exception { SubmittedVariantEntity submittedVariantEntity1 = getSubmittedVariantEntity(ASM_1, PROJECT_1, 1000, 3000000000L, - 5000000000L, NOT_REMAPPED); + 5000000000L, NOT_REMAPPED); mongoTemplate.insert(submittedVariantEntity1, SUBMITTED_VARIANT_COLLECTION); SubmittedVariantEntity submittedVariantEntity2 = getSubmittedVariantEntity(ASM_2, PROJECT_2, 2000, 3000000006L, - 5000000006L, NOT_REMAPPED); + 5000000006L, NOT_REMAPPED); mongoTemplate.insert(submittedVariantEntity2, SUBMITTED_VARIANT_COLLECTION); ClusteredVariantEntity clusteredVariantEntity1 = getClusteredVariantEntity(ASM_1, 1000, 3000000000L); @@ -253,7 +240,7 @@ public void test_variant_clustered_rs_id_exist() throws Exception { //variant remapped from asm1 to asm 2 SubmittedVariantEntity submittedVariantEntity3 = getSubmittedVariantEntity(ASM_2, PROJECT_1, 2000, 3000000000L, - 5000000000L, ASM_1); + 5000000000L, ASM_1); mongoTemplate.insert(submittedVariantEntity3, SUBMITTED_VARIANT_COLLECTION); //asm2 clustered @@ -264,18 +251,18 @@ public void test_variant_clustered_rs_id_exist() throws Exception { //get all submitted variants with assembly asm2 and assert rs id List submittedVariants = mongoTemplate.findAll(SubmittedVariantEntity.class).stream() - .filter(s -> s.getReferenceSequenceAccession() - .equals(ASM_2)) - .collect(Collectors.toList()); + .filter(s -> s.getReferenceSequenceAccession() + .equals(ASM_2)) + .collect(Collectors.toList()); assertEquals(2, submittedVariants.size()); //get all clusteredVariantEntity and check rs id for all List clusteredVariants = mongoTemplate.findAll(ClusteredVariantEntity.class); assertEquals(2, clusteredVariants.size()); List clusteredVariantsASM_1 = clusteredVariants.stream() - .filter(cve->cve.getAssemblyAccession().equals(ASM_1)).collect(Collectors.toList()); + .filter(cve -> cve.getAssemblyAccession().equals(ASM_1)).collect(Collectors.toList()); List clusteredVariantsASM_2 = clusteredVariants.stream() - .filter(cve->cve.getAssemblyAccession().equals(ASM_2)).collect(Collectors.toList()); + .filter(cve -> cve.getAssemblyAccession().equals(ASM_2)).collect(Collectors.toList()); assertEquals(1, clusteredVariantsASM_1.size()); assertEquals(1, clusteredVariantsASM_2.size()); assertEquals(3000000000L, clusteredVariantsASM_1.get(0).getAccession().longValue()); @@ -290,8 +277,8 @@ public void test_variant_clustered_rs_id_exist() throws Exception { SubmittedVariantOperationEntity submittedVariantOperationEntity = submittedVariantOperationEntities.get(0); assertEquals(5000000006L, submittedVariantOperationEntity.getAccession().longValue()); assertEquals(3000000006L, - submittedVariantOperationEntity.getInactiveObjects().get(0).getClusteredVariantAccession() - .longValue()); + submittedVariantOperationEntity.getInactiveObjects().get(0).getClusteredVariantAccession() + .longValue()); assertEquals(EventType.UPDATED, submittedVariantOperationEntity.getEventType()); } @@ -299,12 +286,12 @@ public void test_variant_clustered_rs_id_exist() throws Exception { @DirtiesContext public void test_variant_not_clustered_rs_id_does_not_exist() throws Exception { SubmittedVariantEntity submittedVariantEntity1 = getSubmittedVariantEntity(ASM_1, PROJECT_1, 1000, null, - 5000000000L, NOT_REMAPPED); + 5000000000L, NOT_REMAPPED); mongoTemplate.insert(submittedVariantEntity1, SUBMITTED_VARIANT_COLLECTION); //variant remapped from asm1 to asm2 SubmittedVariantEntity submittedVariantEntity2 = getSubmittedVariantEntity(ASM_2, PROJECT_1, 2000, null, - 5000000000L, ASM_1); + 5000000000L, ASM_1); mongoTemplate.insert(submittedVariantEntity2, SUBMITTED_VARIANT_COLLECTION); //asm2 clustered @@ -320,13 +307,13 @@ public void test_variant_not_clustered_rs_id_does_not_exist() throws Exception { assertEquals(new HashSet<>(Collections.singletonList(3000000000L)), submittedVariants.stream().map(SubmittedVariantEntity::getClusteredVariantAccession) .filter(Objects::nonNull).collect(Collectors.toSet()) - ); + ); assertEquals(new HashSet<>(Collections.singletonList(3000000000L)), submittedVariants.stream().map(SubmittedVariantEntity::getBackPropagatedVariantAccession) .filter(Objects::nonNull).collect(Collectors.toSet()) ); assertTrue(submittedVariants.stream().map(SubmittedVariantEntity::getReferenceSequenceAccession).collect( - Collectors.toList()).containsAll(Arrays.asList(ASM_1, ASM_2))); + Collectors.toList()).containsAll(Arrays.asList(ASM_1, ASM_2))); //get all clusteredVariantEntity and check rs id for all List clusteredVariants = mongoTemplate.findAll(ClusteredVariantEntity.class); @@ -347,25 +334,25 @@ public void test_variant_not_clustered_rs_id_does_not_exist() throws Exception { .collect(Collectors.toList()).containsAll(Arrays.asList(EventType.UPDATED, EventType.RS_BACK_PROPAGATED))); assertEquals("Clustering submitted variant 5000000000 with rs3000000000", - submittedVariantOperationEntities.stream().filter(svoe -> svoe.getInactiveObjects().get(0) - .getReferenceSequenceAccession() - .equals(ASM_2)).findFirst().get() - .getReason()); + submittedVariantOperationEntities.stream().filter(svoe -> svoe.getInactiveObjects().get(0) + .getReferenceSequenceAccession() + .equals(ASM_2)).findFirst().get() + .getReason()); assertEquals("Back-propagating rs3000000000 for submitted variant ss5000000000 after remapping to asm2.", - submittedVariantOperationEntities.stream().filter(svoe -> svoe.getInactiveObjects().get(0) - .getReferenceSequenceAccession() - .equals(ASM_1)).findFirst().get() - .getReason()); + submittedVariantOperationEntities.stream().filter(svoe -> svoe.getInactiveObjects().get(0) + .getReferenceSequenceAccession() + .equals(ASM_1)).findFirst().get() + .getReason()); } @Test @DirtiesContext public void test_variant_not_clustered_rs_id_exist() throws Exception { SubmittedVariantEntity submittedVariantEntity1 = getSubmittedVariantEntity(ASM_1, PROJECT_1, 1000, null, - 5000000000L, NOT_REMAPPED); + 5000000000L, NOT_REMAPPED); mongoTemplate.insert(submittedVariantEntity1, SUBMITTED_VARIANT_COLLECTION); SubmittedVariantEntity submittedVariantEntity2 = getSubmittedVariantEntity(ASM_2, PROJECT_2, 2000, 3000000006L, - 5000000006L, NOT_REMAPPED); + 5000000006L, NOT_REMAPPED); mongoTemplate.insert(submittedVariantEntity2, SUBMITTED_VARIANT_COLLECTION); ClusteredVariantEntity clusteredVariantEntity2 = getClusteredVariantEntity(ASM_2, 2000, 3000000006L); @@ -373,7 +360,7 @@ public void test_variant_not_clustered_rs_id_exist() throws Exception { //variant remapped from asm1 to asm2 SubmittedVariantEntity submittedVariantEntity3 = getSubmittedVariantEntity(ASM_2, PROJECT_1, 2000, null, - 5000000000L, ASM_1); + 5000000000L, ASM_1); mongoTemplate.insert(submittedVariantEntity3, SUBMITTED_VARIANT_COLLECTION); //asm2 clustered @@ -389,7 +376,7 @@ public void test_variant_not_clustered_rs_id_exist() throws Exception { // submittedVariantEntity1 will receive the back-propagated RS assertEquals(submittedVariants.stream().map(SubmittedVariantEntity::getClusteredVariantAccession) .filter(Objects::nonNull).collect(Collectors.toSet()), - new HashSet<>(Collections.singletonList(3000000006L))); + new HashSet<>(Collections.singletonList(3000000006L))); assertEquals(submittedVariants.stream().map(SubmittedVariantEntity::getBackPropagatedVariantAccession) .filter(Objects::nonNull).collect(Collectors.toSet()), new HashSet<>(Collections.singletonList(3000000006L))); @@ -417,41 +404,41 @@ public void test_variant_not_clustered_rs_id_exist() throws Exception { .collect(Collectors.toList()).containsAll(Arrays.asList(EventType.UPDATED, EventType.RS_BACK_PROPAGATED))); assertEquals("Clustering submitted variant 5000000000 with rs3000000006", - submittedVariantOperationEntities.stream().filter(svoe -> svoe.getInactiveObjects().get(0) - .getReferenceSequenceAccession() - .equals(ASM_2)).findFirst().get() - .getReason()); + submittedVariantOperationEntities.stream().filter(svoe -> svoe.getInactiveObjects().get(0) + .getReferenceSequenceAccession() + .equals(ASM_2)).findFirst().get() + .getReason()); assertEquals("Back-propagating rs3000000006 for submitted variant ss5000000000 after remapping to asm2.", - submittedVariantOperationEntities.stream().filter(svoe -> svoe.getInactiveObjects().get(0) - .getReferenceSequenceAccession() - .equals(ASM_1)).findFirst().get() - .getReason()); + submittedVariantOperationEntities.stream().filter(svoe -> svoe.getInactiveObjects().get(0) + .getReferenceSequenceAccession() + .equals(ASM_1)).findFirst().get() + .getReason()); } /** * This test is for the testing the detection of Merge and RS split candidates. * These are already clustered variants, but due to remapping, the variants with same rs id might end up in two * different locations or different types. These needs to be identified for rectifying later. - * + *

* --------------------Before Clustering-------------------- - * - * SubmittedVariantEntity + *

+ * SubmittedVariantEntity * SS RS ASM STUDY CONTIG POS REF ALT * 500 306 ASM1 PRJEB1 Chr1 1000 A T (original) * 501 306 ASM1 PRJEB2 Chr1 1000 A T (original) * 500 306 ASM2 PRJEB1 Chr1 1000 A T (remapped) * 501 306 ASM2 PRJEB2 Chr1 1500 A T (remapped) - * - * ClusteredVariantEntity + *

+ * ClusteredVariantEntity * RS HASH ASM POS CONTIG TYPE * 306 ASM1_Chr1_1000_SNV ASM1 1000 Chr1 SNV - * + *

* SS id 500 and 501 has same RS because of remapping, but they are now at different positions and can't have same RS id. * These needs to be identified and stored in submittedVariantOperationEntity table for rectification. - * + *

* --------------------After Detection-------------------- - * - * SubmittedVariantOperationEntity + *

+ * SubmittedVariantOperationEntity * ACCESSION EVENT_TYPE REASON INACTIVe_OBJECTS * 306 RS_SPLIT_CANDIDATE Hash mismatch with 306 {ss-500 and ss-501} */ @@ -460,68 +447,68 @@ public void test_variant_not_clustered_rs_id_exist() throws Exception { public void test_detect_RS_MERGE_AND_SPLIT_different_batches() throws Exception { //Batch 1 SubmittedVariantEntity submittedVariantEntityRemapped1 = getSubmittedVariantEntity(ASM_2, "project_1", - 5000000001L, 3000000001L, 1000,"A", "T", ASM_1); + 5000000001L, 3000000001L, 1000, "A", "T", ASM_1); mongoTemplate.insert(submittedVariantEntityRemapped1, SUBMITTED_VARIANT_COLLECTION); SubmittedVariantEntity submittedVariantEntityRemapped2 = getSubmittedVariantEntity(ASM_2, "project_2", - 5000000002L, 3000000003L, 2000,"A", "T", ASM_1); + 5000000002L, 3000000003L, 2000, "A", "T", ASM_1); mongoTemplate.insert(submittedVariantEntityRemapped2, SUBMITTED_VARIANT_COLLECTION); - clusteringWriterPreMergeAndSplit.write(Arrays.asList(submittedVariantEntityRemapped1, submittedVariantEntityRemapped2)); + clusteringWriterPreMergeAndSplit.write(Chunk.of(submittedVariantEntityRemapped1, submittedVariantEntityRemapped2)); //Batch 2 - split event SubmittedVariantEntity submittedVariantEntityRemapped3 = getSubmittedVariantEntity(ASM_2, "project_3", - 5000000003L, 3000000001L, 3000,"A", "T", ASM_1); + 5000000003L, 3000000001L, 3000, "A", "T", ASM_1); mongoTemplate.insert(submittedVariantEntityRemapped3, SUBMITTED_VARIANT_COLLECTION); - clusteringWriterPreMergeAndSplit.write(Arrays.asList(submittedVariantEntityRemapped3)); + clusteringWriterPreMergeAndSplit.write(Chunk.of(submittedVariantEntityRemapped3)); //Batch 3 - split event SubmittedVariantEntity submittedVariantEntityRemapped4 = getSubmittedVariantEntity(ASM_2, "project_4", - 5000000004L, 3000000001L, 4000,"A", "T", ASM_1); + 5000000004L, 3000000001L, 4000, "A", "T", ASM_1); mongoTemplate.insert(submittedVariantEntityRemapped4, SUBMITTED_VARIANT_COLLECTION); - clusteringWriterPreMergeAndSplit.write(Arrays.asList(submittedVariantEntityRemapped4)); + clusteringWriterPreMergeAndSplit.write(Chunk.of(submittedVariantEntityRemapped4)); //Batch 4 - merge event SubmittedVariantEntity submittedVariantEntityRemapped5 = getSubmittedVariantEntity(ASM_2, "project_5", - 5000000005L, 3000000002L, 2000,"A", "T", ASM_1); + 5000000005L, 3000000002L, 2000, "A", "T", ASM_1); mongoTemplate.insert(submittedVariantEntityRemapped5, SUBMITTED_VARIANT_COLLECTION); - clusteringWriterPreMergeAndSplit.write(Arrays.asList(submittedVariantEntityRemapped5)); + clusteringWriterPreMergeAndSplit.write(Chunk.of(submittedVariantEntityRemapped5)); //Batch 5 - merge event SubmittedVariantEntity submittedVariantEntityRemapped6 = getSubmittedVariantEntity(ASM_2, "project_6", - 5000000006L, 3000000002L, 2000,"A", "T", ASM_1); + 5000000006L, 3000000002L, 2000, "A", "T", ASM_1); mongoTemplate.insert(submittedVariantEntityRemapped6, SUBMITTED_VARIANT_COLLECTION); - clusteringWriterPreMergeAndSplit.write(Arrays.asList(submittedVariantEntityRemapped6)); + clusteringWriterPreMergeAndSplit.write(Chunk.of(submittedVariantEntityRemapped6)); //Assert assertMergeAndSplitCandidatesScenarios(Arrays.asList(submittedVariantEntityRemapped1, - submittedVariantEntityRemapped2,submittedVariantEntityRemapped3,submittedVariantEntityRemapped4, - submittedVariantEntityRemapped5,submittedVariantEntityRemapped6)); + submittedVariantEntityRemapped2, submittedVariantEntityRemapped3, submittedVariantEntityRemapped4, + submittedVariantEntityRemapped5, submittedVariantEntityRemapped6)); } @Test @DirtiesContext public void test_detect_RS_MERGE_AND_SPLIT_same_batch() throws Exception { SubmittedVariantEntity submittedVariantEntityRemapped1 = getSubmittedVariantEntity(ASM_2, "project_1", - 5000000001L, 3000000001L, 1000,"A", "T", ASM_1); + 5000000001L, 3000000001L, 1000, "A", "T", ASM_1); mongoTemplate.insert(submittedVariantEntityRemapped1, SUBMITTED_VARIANT_COLLECTION); SubmittedVariantEntity submittedVariantEntityRemapped2 = getSubmittedVariantEntity(ASM_2, "project_2", - 5000000002L, 3000000003L, 2000,"A", "T", ASM_1); + 5000000002L, 3000000003L, 2000, "A", "T", ASM_1); mongoTemplate.insert(submittedVariantEntityRemapped2, SUBMITTED_VARIANT_COLLECTION); SubmittedVariantEntity submittedVariantEntityRemapped3 = getSubmittedVariantEntity(ASM_2, "project_3", - 5000000003L, 3000000001L, 3000,"A", "T", ASM_1); + 5000000003L, 3000000001L, 3000, "A", "T", ASM_1); mongoTemplate.insert(submittedVariantEntityRemapped3, SUBMITTED_VARIANT_COLLECTION); SubmittedVariantEntity submittedVariantEntityRemapped4 = getSubmittedVariantEntity(ASM_2, "project_4", - 5000000004L, 3000000001L, 4000,"A", "T", ASM_1); + 5000000004L, 3000000001L, 4000, "A", "T", ASM_1); mongoTemplate.insert(submittedVariantEntityRemapped4, SUBMITTED_VARIANT_COLLECTION); SubmittedVariantEntity submittedVariantEntityRemapped5 = getSubmittedVariantEntity(ASM_2, "project_5", - 5000000005L, 3000000002L, 2000,"A", "T", ASM_1); + 5000000005L, 3000000002L, 2000, "A", "T", ASM_1); mongoTemplate.insert(submittedVariantEntityRemapped5, SUBMITTED_VARIANT_COLLECTION); SubmittedVariantEntity submittedVariantEntityRemapped6 = getSubmittedVariantEntity(ASM_2, "project_6", - 5000000006L, 3000000002L, 2000,"A", "T", ASM_1); + 5000000006L, 3000000002L, 2000, "A", "T", ASM_1); mongoTemplate.insert(submittedVariantEntityRemapped6, SUBMITTED_VARIANT_COLLECTION); List submittedVariantEntityList = new ArrayList<>(); @@ -531,44 +518,44 @@ public void test_detect_RS_MERGE_AND_SPLIT_same_batch() throws Exception { submittedVariantEntityList.add(submittedVariantEntityRemapped4); submittedVariantEntityList.add(submittedVariantEntityRemapped5); submittedVariantEntityList.add(submittedVariantEntityRemapped6); - clusteringWriterPreMergeAndSplit.write(Arrays.asList(submittedVariantEntityRemapped1, submittedVariantEntityRemapped2, - submittedVariantEntityRemapped3, submittedVariantEntityRemapped4, - submittedVariantEntityRemapped5, submittedVariantEntityRemapped6)); + clusteringWriterPreMergeAndSplit.write(Chunk.of(submittedVariantEntityRemapped1, submittedVariantEntityRemapped2, + submittedVariantEntityRemapped3, submittedVariantEntityRemapped4, + submittedVariantEntityRemapped5, submittedVariantEntityRemapped6)); //Assert assertMergeAndSplitCandidatesScenarios(Arrays.asList(submittedVariantEntityRemapped1, - submittedVariantEntityRemapped2,submittedVariantEntityRemapped3,submittedVariantEntityRemapped4, - submittedVariantEntityRemapped5,submittedVariantEntityRemapped6)); + submittedVariantEntityRemapped2, submittedVariantEntityRemapped3, submittedVariantEntityRemapped4, + submittedVariantEntityRemapped5, submittedVariantEntityRemapped6)); } - private void assertMergeAndSplitCandidatesScenarios(List submittedVariantEntities){ + private void assertMergeAndSplitCandidatesScenarios(List submittedVariantEntities) { //assert clusteredVariantEntity List clusteredVariantEntities = mongoTemplate.findAll(ClusteredVariantEntity.class); assertEquals(4, clusteredVariantEntities.size()); - assertEquals(1,clusteredVariantEntities.stream().filter(cve->cve.getAccession().equals(3000000003L)).count()); - assertEquals(3,clusteredVariantEntities.stream().filter(cve->cve.getAccession().equals(3000000001L)).count()); + assertEquals(1, clusteredVariantEntities.stream().filter(cve -> cve.getAccession().equals(3000000003L)).count()); + assertEquals(3, clusteredVariantEntities.stream().filter(cve -> cve.getAccession().equals(3000000001L)).count()); //assert merge event SubmittedVariantOperationEntity submittedVariantOperationEntityMerge = mongoTemplate.findAll(SubmittedVariantOperationEntity.class).stream() - .filter(svoe->svoe.getEventType().equals(EventType.RS_MERGE_CANDIDATES)) + .filter(svoe -> svoe.getEventType().equals(EventType.RS_MERGE_CANDIDATES)) .collect(Collectors.toList()).get(0); assertEquals(EventType.RS_MERGE_CANDIDATES, submittedVariantOperationEntityMerge.getEventType()); - assertEquals(3000000003L,submittedVariantOperationEntityMerge.getAccession().longValue()); + assertEquals(3000000003L, submittedVariantOperationEntityMerge.getAccession().longValue()); assertEquals("RS mismatch with 3000000003", submittedVariantOperationEntityMerge.getReason()); assertEquals(3, submittedVariantOperationEntityMerge.getInactiveObjects().size()); assertTrue(submittedVariantOperationEntityMerge.getInactiveObjects().containsAll( Arrays.asList(new SubmittedVariantInactiveEntity(submittedVariantEntities.get(1)), new SubmittedVariantInactiveEntity(submittedVariantEntities.get(4)), - new SubmittedVariantInactiveEntity(submittedVariantEntities.get(5))))); + new SubmittedVariantInactiveEntity(submittedVariantEntities.get(5))))); //assert split event SubmittedVariantOperationEntity submittedVariantOperationEntitySplit = mongoTemplate.findAll(SubmittedVariantOperationEntity.class).stream() - .filter(svoe->svoe.getEventType().equals(EventType.RS_SPLIT_CANDIDATES)) + .filter(svoe -> svoe.getEventType().equals(EventType.RS_SPLIT_CANDIDATES)) .collect(Collectors.toList()).get(0); assertEquals(EventType.RS_SPLIT_CANDIDATES, submittedVariantOperationEntitySplit.getEventType()); - assertEquals(3000000001L,submittedVariantOperationEntitySplit.getAccession().longValue()); + assertEquals(3000000001L, submittedVariantOperationEntitySplit.getAccession().longValue()); assertEquals("Hash mismatch with 3000000001", submittedVariantOperationEntitySplit.getReason()); assertEquals(3, submittedVariantOperationEntitySplit.getInactiveObjects().size()); assertTrue(submittedVariantOperationEntitySplit.getInactiveObjects().containsAll( @@ -580,7 +567,7 @@ private void assertMergeAndSplitCandidatesScenarios(List private SubmittedVariantEntity getSubmittedVariantEntity(String assembly, String project, long ss, Long rs, long start, String ref, String alt, String remappedFrom) { SubmittedVariant submittedClustered = new SubmittedVariant(assembly, TAXONOMY, project, CONTIG, start, ref, alt, - rs); + rs); String hash1 = hashingFunction.apply(submittedClustered); return new SubmittedVariantEntity(ss, hash1, submittedClustered, 1, remappedFrom, LocalDateTime.now(), null); } @@ -602,41 +589,41 @@ private ClusteredVariantEntity getClusteredVariantEntity(String assembly, long s private void clusterVariants(List submittedVariantEntities) throws Exception { - clusteringWriterPreMergeAndSplit.write(submittedVariantEntities); + clusteringWriterPreMergeAndSplit.write(new Chunk<>(submittedVariantEntities)); List mergeCandidates = new ArrayList<>(); List splitCandidates = new ArrayList<>(); SubmittedVariantOperationEntity tempSVO; rsMergeCandidatesReader.open(new ExecutionContext()); - while((tempSVO = rsMergeCandidatesReader.read()) != null) { + while ((tempSVO = rsMergeCandidatesReader.read()) != null) { mergeCandidates.add(tempSVO); } rsSplitCandidatesReader.open(new ExecutionContext()); - while((tempSVO = rsSplitCandidatesReader.read()) != null) { + while ((tempSVO = rsSplitCandidatesReader.read()) != null) { splitCandidates.add(tempSVO); } - rsMergeWriter.write(mergeCandidates); - rsSplitWriter.write(splitCandidates); + rsMergeWriter.write(new Chunk<>(mergeCandidates)); + rsSplitWriter.write(new Chunk<>(splitCandidates)); ClusteringMongoReader unclusteredVariantsReader = new ClusteringMongoReader(this.mongoTemplate, ASM_2, 100, - false); + false); unclusteredVariantsReader.initializeReader(); List unclusteredVariants = new ArrayList<>(); SubmittedVariantEntity tempSV; - while((tempSV = unclusteredVariantsReader.read()) != null) { + while ((tempSV = unclusteredVariantsReader.read()) != null) { unclusteredVariants.add(tempSV); } unclusteredVariantsReader.close(); // Cluster non-clustered variants - clusteringWriterPostMergeAndSplit.write(unclusteredVariants); + clusteringWriterPostMergeAndSplit.write(new Chunk<>(unclusteredVariants)); // Spring has a mandatory requirement of even small functionality being writers. // To satisfy that, we pass in a dummy object to invoke the writer // which basically clears the merge and split operations after they were processed above - clearRSMergeAndSplitCandidates.write(Collections.singletonList(new Object())); + clearRSMergeAndSplitCandidates.write(Chunk.of(new Object())); ClusteringMongoReader originalAssemblyUnclusteredVariantsReader = new ClusteringMongoReader(this.mongoTemplate, ASM_1, 100, false); originalAssemblyUnclusteredVariantsReader.initializeReader(); - while((tempSV = originalAssemblyUnclusteredVariantsReader.read()) != null) { - backPropagatedRSWriter.write(Collections.singletonList(tempSV)); + while ((tempSV = originalAssemblyUnclusteredVariantsReader.read()) != null) { + backPropagatedRSWriter.write(Chunk.of(tempSV)); } } } diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/clustering_writer/ReuseAccessionClusteringWriterTest.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/clustering_writer/ReuseAccessionClusteringWriterTest.java index 6784e3842..c1b132875 100644 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/clustering_writer/ReuseAccessionClusteringWriterTest.java +++ b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/clustering_writer/ReuseAccessionClusteringWriterTest.java @@ -15,15 +15,13 @@ */ package uk.ac.ebi.eva.accession.clustering.batch.io.clustering_writer; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mockito; import org.springframework.batch.core.JobExecution; +import org.springframework.batch.item.Chunk; import org.springframework.batch.item.ExecutionContext; import org.springframework.batch.item.ItemWriter; import org.springframework.beans.factory.annotation.Autowired; @@ -36,17 +34,15 @@ import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; - +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.ampt2d.commons.accession.hashing.SHA1HashingFunction; - import uk.ac.ebi.eva.accession.clustering.batch.io.ClusteringMongoReader; import uk.ac.ebi.eva.accession.clustering.batch.io.ClusteringWriter; import uk.ac.ebi.eva.accession.clustering.batch.io.RSSplitWriter; import uk.ac.ebi.eva.accession.clustering.metric.ClusteringMetric; import uk.ac.ebi.eva.accession.clustering.parameters.InputParameters; import uk.ac.ebi.eva.accession.clustering.test.configuration.BatchTestConfiguration; -import uk.ac.ebi.eva.accession.clustering.test.rule.FixSpringMongoDbRule; +import uk.ac.ebi.eva.accession.clustering.test.configuration.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.ClusteredVariantAccessioningConfiguration; import uk.ac.ebi.eva.accession.core.model.ClusteredVariant; import uk.ac.ebi.eva.accession.core.model.IClusteredVariant; @@ -61,6 +57,7 @@ import uk.ac.ebi.eva.accession.core.service.nonhuman.ClusteredVariantAccessioningService; import uk.ac.ebi.eva.accession.core.summary.ClusteredVariantSummaryFunction; import uk.ac.ebi.eva.accession.core.summary.SubmittedVariantSummaryFunction; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.commons.core.models.VariantType; import uk.ac.ebi.eva.commons.mongodb.readers.MongoDbCursorItemReader; import uk.ac.ebi.eva.metrics.metric.MetricCompute; @@ -70,8 +67,8 @@ import java.util.List; import java.util.function.Function; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.springframework.data.mongodb.core.query.Criteria.where; import static uk.ac.ebi.eva.accession.clustering.batch.io.clustering_writer.ClusteringAssertions.assertClusteringCounts; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.BACK_PROPAGATED_RS_WRITER; @@ -85,19 +82,17 @@ /** * This class handles some scenarios of ClusteringWriter where an existing RS is reused. - * + *

* The scenarios tested here are those about either reusing the clustered variant accessions (RSs) that a submitted * variant provides, or reusing an RS that exists in the database because it's equivalent (shares identifying fields). * Other test classes in this folder take care of other scenarios. */ -@RunWith(SpringRunner.class) +@ExtendWith(SpringExtension.class) @EnableAutoConfiguration -@ContextConfiguration(classes = {ClusteredVariantAccessioningConfiguration.class, BatchTestConfiguration.class}) +@ContextConfiguration(classes = {ClusteredVariantAccessioningConfiguration.class, BatchTestConfiguration.class, + MongoTestConfiguration.class}) @TestPropertySource("classpath:clustering-writer-test.properties") -public class ReuseAccessionClusteringWriterTest { - - private static final String TEST_DB = "test-db"; - +public class ReuseAccessionClusteringWriterTest extends MongoTestContainerHelper { private static String ASM_1; private static String ASM_2; @@ -160,11 +155,7 @@ public class ReuseAccessionClusteringWriterTest { private Function clusteredHashingFunction; - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName(TEST_DB).build()); - - @Before + @BeforeEach public void setUp() { ASM_1 = inputParameters.getRemappedFrom(); ASM_2 = inputParameters.getAssemblyAccession(); @@ -179,7 +170,7 @@ public void setUp() { clusteringWriterPostMergeAndSplit.setJobExecution(jobExecution); } - @After + @AfterEach public void tearDown() { mongoTemplate.getDb().drop(); metricCompute.clearCount(); @@ -233,11 +224,11 @@ private DbsnpSubmittedVariantEntity createDbsnpSubmittedVariantEntity(String ass SubmittedVariant submittedClustered = new SubmittedVariant(assembly, 1000, "project", "1", 100L, "T", "A", rs); String hash1 = hashingFunction.apply(submittedClustered); DbsnpSubmittedVariantEntity dbsnpSubmittedVariantEntity = new DbsnpSubmittedVariantEntity(ss, hash1, - submittedClustered, - 1); + submittedClustered, + 1); dbsnpSubmittedVariantEntity.setRemappedFrom(remappedFromAssembly); mongoTemplate.save(dbsnpSubmittedVariantEntity, - mongoTemplate.getCollectionName(DbsnpSubmittedVariantEntity.class)); + mongoTemplate.getCollectionName(DbsnpSubmittedVariantEntity.class)); return dbsnpSubmittedVariantEntity; } @@ -308,41 +299,41 @@ public void reuse_dbsnp_clustered_accession_when_clustering_an_eva_submitted_var private void clusterVariants(List submittedVariantEntities) throws Exception { - clusteringWriterPreMergeAndSplit.write(submittedVariantEntities); + clusteringWriterPreMergeAndSplit.write(new Chunk<>(submittedVariantEntities)); List mergeCandidates = new ArrayList<>(); List splitCandidates = new ArrayList<>(); SubmittedVariantOperationEntity tempSVO; rsMergeCandidatesReader.open(new ExecutionContext()); - while((tempSVO = rsMergeCandidatesReader.read()) != null) { + while ((tempSVO = rsMergeCandidatesReader.read()) != null) { mergeCandidates.add(tempSVO); } rsSplitCandidatesReader.open(new ExecutionContext()); - while((tempSVO = rsSplitCandidatesReader.read()) != null) { + while ((tempSVO = rsSplitCandidatesReader.read()) != null) { splitCandidates.add(tempSVO); } - rsMergeWriter.write(mergeCandidates); - rsSplitWriter.write(splitCandidates); + rsMergeWriter.write(new Chunk<>(mergeCandidates)); + rsSplitWriter.write(new Chunk<>(splitCandidates)); ClusteringMongoReader unclusteredVariantsReader = new ClusteringMongoReader(this.mongoTemplate, ASM_2, 100, - false); + false); unclusteredVariantsReader.initializeReader(); List unclusteredVariants = new ArrayList<>(); SubmittedVariantEntity tempSV; - while((tempSV = unclusteredVariantsReader.read()) != null) { + while ((tempSV = unclusteredVariantsReader.read()) != null) { unclusteredVariants.add(tempSV); } unclusteredVariantsReader.close(); // Cluster non-clustered variants - clusteringWriterPostMergeAndSplit.write(unclusteredVariants); + clusteringWriterPostMergeAndSplit.write(new Chunk<>(unclusteredVariants)); // Spring has a mandatory requirement of even small functionality being writers. // To satisfy that, we pass in a dummy object to invoke the writer // which basically clears the merge and split operations after they were processed above - clearRSMergeAndSplitCandidates.write(Collections.singletonList(new Object())); + clearRSMergeAndSplitCandidates.write(Chunk.of(new Object())); ClusteringMongoReader originalAssemblyUnclusteredVariantsReader = new ClusteringMongoReader(this.mongoTemplate, ASM_1, 100, false); originalAssemblyUnclusteredVariantsReader.initializeReader(); - while((tempSV = originalAssemblyUnclusteredVariantsReader.read()) != null) { - backPropagatedRSWriter.write(Collections.singletonList(tempSV)); + while ((tempSV = originalAssemblyUnclusteredVariantsReader.read()) != null) { + backPropagatedRSWriter.write(Chunk.of(tempSV)); } } } diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/processors/VariantToSubmittedVariantEntityProcessorTest.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/processors/VariantToSubmittedVariantEntityProcessorTest.java index fb7b037db..b2af4d2e8 100644 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/processors/VariantToSubmittedVariantEntityProcessorTest.java +++ b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/processors/VariantToSubmittedVariantEntityProcessorTest.java @@ -15,8 +15,8 @@ */ package uk.ac.ebi.eva.accession.clustering.batch.processors; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import uk.ac.ebi.ampt2d.commons.accession.hashing.SHA1HashingFunction; import uk.ac.ebi.eva.accession.core.model.ISubmittedVariant; import uk.ac.ebi.eva.accession.core.model.SubmittedVariant; @@ -28,8 +28,8 @@ import java.util.function.Function; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThrows; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; public class VariantToSubmittedVariantEntityProcessorTest { @@ -43,7 +43,7 @@ public class VariantToSubmittedVariantEntityProcessorTest { private VariantToSubmittedVariantEntityProcessor processor; - @Before + @BeforeEach public void setUp() { processor = new VariantToSubmittedVariantEntityProcessor(ASSEMBLY_ACCESSION); hashingFunction = new SubmittedVariantSummaryFunction().andThen(new SHA1HashingFunction()); diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/ClusteringVariantJobConfigurationTest.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/ClusteringVariantJobConfigurationTest.java index 391aada9e..f9fc8abac 100644 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/ClusteringVariantJobConfigurationTest.java +++ b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/ClusteringVariantJobConfigurationTest.java @@ -15,40 +15,38 @@ */ package uk.ac.ebi.eva.accession.clustering.configuration.batch.jobs; -import com.lordofthejars.nosqlunit.annotation.UsingDataSet; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.batch.core.BatchStatus; import org.springframework.batch.core.JobExecution; import org.springframework.batch.core.StepExecution; import org.springframework.batch.test.JobLauncherTestUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.context.ApplicationContext; +import org.springframework.core.io.ResourceLoader; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.http.HttpMethod; import org.springframework.http.HttpStatus; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import org.springframework.test.web.client.ExpectedCount; import org.springframework.test.web.client.MockRestServiceServer; - import org.springframework.web.client.RestTemplate; import uk.ac.ebi.eva.accession.clustering.configuration.batch.io.RSMergeAndSplitCandidatesReaderConfiguration; import uk.ac.ebi.eva.accession.clustering.parameters.InputParameters; import uk.ac.ebi.eva.accession.clustering.test.DatabaseState; +import uk.ac.ebi.eva.accession.clustering.test.configuration.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.accession.clustering.test.configuration.BatchTestConfiguration; -import uk.ac.ebi.eva.accession.clustering.test.rule.FixSpringMongoDbRule; +import uk.ac.ebi.eva.accession.clustering.test.configuration.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantInactiveEntity; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantOperationEntity; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; +import uk.ac.ebi.eva.accession.core.utils.MongoTestDataLoader; import uk.ac.ebi.eva.metrics.count.CountServiceParameters; import java.net.URI; @@ -58,7 +56,7 @@ import java.util.List; import java.util.stream.Collectors; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.springframework.test.web.client.match.MockRestRequestMatchers.method; import static org.springframework.test.web.client.match.MockRestRequestMatchers.requestTo; import static org.springframework.test.web.client.response.MockRestResponseCreators.withStatus; @@ -78,13 +76,11 @@ import static uk.ac.ebi.eva.accession.clustering.test.configuration.BatchTestConfiguration.JOB_LAUNCHER_RS_ACCESSION_RECOVERY; import static uk.ac.ebi.eva.accession.clustering.test.configuration.BatchTestConfiguration.JOB_LAUNCHER_STUDY_FROM_MONGO; -@RunWith(SpringRunner.class) -@ContextConfiguration(classes = {BatchTestConfiguration.class}) +@ExtendWith(SpringExtension.class) +@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class, + BatchJobRepositoryTestConfiguration.class}) @TestPropertySource("classpath:clustering-writer-test.properties") -public class ClusteringVariantJobConfigurationTest { - - private static final String TEST_DB = "test-db"; - +public class ClusteringVariantJobConfigurationTest extends MongoTestContainerHelper { @Autowired @Qualifier(JOB_LAUNCHER_FROM_MONGO) private JobLauncherTestUtils jobLauncherTestUtilsFromMongo; @@ -100,9 +96,8 @@ public class ClusteringVariantJobConfigurationTest { @Autowired private MongoTemplate mongoTemplate; - //Required by nosql-unit @Autowired - private ApplicationContext applicationContext; + private ResourceLoader resourceLoader; @Autowired private InputParameters inputParameters; @@ -118,11 +113,7 @@ public class ClusteringVariantJobConfigurationTest { private final String URL_PATH_SAVE_COUNT = "/v1/bulk/count"; - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName(TEST_DB).build()); - - @Before + @BeforeEach public void init() throws Exception { mongoTemplate.getDb().drop(); mockServer = MockRestServiceServer.createServer(restTemplate); @@ -131,15 +122,16 @@ public void init() throws Exception { .andRespond(withStatus(HttpStatus.OK)); } - @After + @AfterEach public void tearDown() { mongoTemplate.getDb().drop(); } @Test @DirtiesContext - @UsingDataSet(locations = {"/test-data/submittedVariantEntityMongoReader.json"}) public void jobFromMongo() throws Exception { + new MongoTestDataLoader(mongoTemplate, resourceLoader).load("/test-data/submittedVariantEntityMongoReader.json"); + JobExecution jobExecution = jobLauncherTestUtilsFromMongo.launchJob(); List expectedSteps = new ArrayList<>(); expectedSteps.add(CLUSTERING_CLUSTERED_VARIANTS_FROM_MONGO_STEP); @@ -156,8 +148,9 @@ public void jobFromMongo() throws Exception { @Test @DirtiesContext - @UsingDataSet(locations = {"/test-data/submittedVariantEntityStudyReader.json"}) public void studyJobFromMongo() throws Exception { + new MongoTestDataLoader(mongoTemplate, resourceLoader).load("/test-data/submittedVariantEntityStudyReader.json"); + JobExecution jobExecution = jobLauncherTestUtilsStudyFromMongo.launchJob(); List expectedSteps = new ArrayList<>(); expectedSteps.add(STUDY_CLUSTERING_STEP); @@ -168,8 +161,9 @@ public void studyJobFromMongo() throws Exception { @Test @DirtiesContext - @UsingDataSet(locations = {"/test-data/submittedVariantEntityStudyReader.json"}) public void testStudyJobFromMongoIdempotent() throws Exception { + new MongoTestDataLoader(mongoTemplate, resourceLoader).load("/test-data/submittedVariantEntityStudyReader.json"); + jobLauncherTestUtilsStudyFromMongo.launchJob(); DatabaseState databaseStateAfterFirstRun = DatabaseState.getCurrentDatabaseState(mongoTemplate); jobLauncherTestUtilsStudyFromMongo.launchJob(); @@ -192,17 +186,17 @@ public void assertDataThatExceedsChunkSizeIsFullyProcessed() throws Exception { createMergeCandidateEntriesThatExceedChunkSize(numOperations); JobExecution jobExecution = jobLauncherTestUtilsFromMongo.launchJob(); assertEquals(numOperations, - jobExecution.getStepExecutions() - .stream() - .filter(stepExecution -> stepExecution.getStepName() - .equals(PROCESS_RS_SPLIT_CANDIDATES_STEP)) - .findFirst().get().getReadCount()); + jobExecution.getStepExecutions() + .stream() + .filter(stepExecution -> stepExecution.getStepName() + .equals(PROCESS_RS_SPLIT_CANDIDATES_STEP)) + .findFirst().get().getReadCount()); assertEquals(numOperations, - jobExecution.getStepExecutions() - .stream() - .filter(stepExecution -> stepExecution.getStepName() - .equals(PROCESS_RS_MERGE_CANDIDATES_STEP)) - .findFirst().get().getReadCount()); + jobExecution.getStepExecutions() + .stream() + .filter(stepExecution -> stepExecution.getStepName() + .equals(PROCESS_RS_MERGE_CANDIDATES_STEP)) + .findFirst().get().getReadCount()); } @Test @@ -224,11 +218,11 @@ private void createMergeCandidateEntriesThatExceedChunkSize(int numSplitCandidat for (int i = 0; i < numSplitCandidateOperations; i++) { SubmittedVariantOperationEntity mergeOperation = new SubmittedVariantOperationEntity(); mergeOperation.fill(RSMergeAndSplitCandidatesReaderConfiguration.MERGE_CANDIDATES_EVENT_TYPE, - ss1.getAccession(), null, "Mock merge candidate", - Arrays.asList(ss1, ss2)); + ss1.getAccession(), null, "Mock merge candidate", + Arrays.asList(ss1, ss2)); mergeOperation.setId(MERGE_CANDIDATE_ID_PREFIX + "_" + ss1.getReferenceSequenceAccession() + "_" + i); mongoTemplate.insert(mergeOperation, - mongoTemplate.getCollectionName(SubmittedVariantOperationEntity.class)); + mongoTemplate.getCollectionName(SubmittedVariantOperationEntity.class)); } } @@ -241,20 +235,20 @@ private void createSplitCandidateEntriesThatExceedChunkSize(int numSplitCandidat for (int i = 0; i < numSplitCandidateOperations; i++) { SubmittedVariantOperationEntity splitOperation = new SubmittedVariantOperationEntity(); splitOperation.fill(RSMergeAndSplitCandidatesReaderConfiguration.SPLIT_CANDIDATES_EVENT_TYPE, - ss3.getAccession(), null, "Mock split candidate", - Arrays.asList(ss3, ss4)); + ss3.getAccession(), null, "Mock split candidate", + Arrays.asList(ss3, ss4)); splitOperation.setId(SPLIT_CANDIDATE_ID_PREFIX + "_" + ss3.getReferenceSequenceAccession() + "_" + i); mongoTemplate.insert(splitOperation, - mongoTemplate.getCollectionName(SubmittedVariantOperationEntity.class)); + mongoTemplate.getCollectionName(SubmittedVariantOperationEntity.class)); } } private SubmittedVariantEntity createSSWithLocus(Long ssAccession, Long rsAccession, Long start, String reference, String alternate) { return new SubmittedVariantEntity(ssAccession, "hash" + ssAccession, inputParameters.getAssemblyAccession(), - 60711, - "PRJ1", "chr1", start, reference, alternate, rsAccession, false, false, false, - false, 1); + 60711, + "PRJ1", "chr1", start, reference, alternate, rsAccession, false, false, false, + false, 1); } } \ No newline at end of file diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/RSAccessionRecoveryTest.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/RSAccessionRecoveryTest.java index dfbbfa08f..7df9926c5 100644 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/RSAccessionRecoveryTest.java +++ b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/RSAccessionRecoveryTest.java @@ -16,45 +16,64 @@ * limitations under the License. */ -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.batch.core.BatchStatus; import org.springframework.batch.core.JobExecution; import org.springframework.batch.test.JobLauncherTestUtils; import org.springframework.batch.test.context.SpringBatchTest; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.ampt2d.commons.accession.persistence.jpa.monotonic.entities.ContiguousIdBlock; import uk.ac.ebi.ampt2d.commons.accession.persistence.jpa.monotonic.repositories.ContiguousIdBlockRepository; +import uk.ac.ebi.eva.accession.clustering.test.configuration.BatchJobRepositoryTestConfiguration; +import uk.ac.ebi.eva.accession.clustering.test.configuration.MongoTestConfiguration; import uk.ac.ebi.eva.accession.clustering.test.configuration.RSAccessionRecoveryTestConfiguration; import uk.ac.ebi.eva.accession.core.model.ClusteredVariant; import uk.ac.ebi.eva.accession.core.model.eva.ClusteredVariantEntity; import uk.ac.ebi.eva.accession.core.repository.nonhuman.eva.ClusteredVariantAccessioningRepository; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.commons.core.models.VariantType; import java.time.LocalDateTime; import java.util.ArrayList; import java.util.List; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import static uk.ac.ebi.eva.accession.clustering.test.configuration.RSAccessionRecoveryTestConfiguration.JOB_LAUNCHER_RS_ACCESSION_RECOVERY; -@RunWith(SpringRunner.class) -@ContextConfiguration(classes = {RSAccessionRecoveryTestConfiguration.class,}) +@ExtendWith(SpringExtension.class) +@ContextConfiguration(classes = {RSAccessionRecoveryTestConfiguration.class, MongoTestConfiguration.class, BatchJobRepositoryTestConfiguration.class}) @TestPropertySource("classpath:rs-accession-recovery.properties") @SpringBatchTest -public class RSAccessionRecoveryTest { +public class RSAccessionRecoveryTest extends MongoTestContainerHelper { @Autowired private ContiguousIdBlockRepository blockRepository; @Autowired private ClusteredVariantAccessioningRepository mongoRepository; + @Autowired + private MongoTemplate mongoTemplate; + + @BeforeEach + public void setUp() { + mongoTemplate.getDb().drop(); + } + + @AfterEach + public void tearDown() { + mongoTemplate.getDb().drop(); + } + @Autowired @Qualifier(JOB_LAUNCHER_RS_ACCESSION_RECOVERY) private JobLauncherTestUtils jobLauncherTestUtilsMonotonicAccessionRecoveryAgent; diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/qc/DuplicateRSAccQCStepConfigurationTest.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/qc/DuplicateRSAccQCStepConfigurationTest.java index 98d2abc28..d3b308691 100644 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/qc/DuplicateRSAccQCStepConfigurationTest.java +++ b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/qc/DuplicateRSAccQCStepConfigurationTest.java @@ -15,32 +15,30 @@ */ package uk.ac.ebi.eva.accession.clustering.configuration.batch.jobs.qc; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; -import com.mongodb.MongoClient; -import gherkin.deps.com.google.gson.Gson; -import gherkin.deps.com.google.gson.GsonBuilder; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; import org.apache.commons.lang3.tuple.Pair; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.batch.core.BatchStatus; import org.springframework.batch.core.JobExecution; import org.springframework.batch.test.JobLauncherTestUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.context.ApplicationContext; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.ampt2d.commons.accession.hashing.SHA1HashingFunction; import uk.ac.ebi.eva.accession.clustering.batch.io.qc.DuplicateRSAccQCResult; +import uk.ac.ebi.eva.accession.clustering.test.configuration.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.accession.clustering.test.configuration.BatchTestConfiguration; import uk.ac.ebi.eva.accession.clustering.test.configuration.MongoTestConfiguration; -import uk.ac.ebi.eva.accession.clustering.test.rule.FixSpringMongoDbRule; import uk.ac.ebi.eva.accession.core.model.ClusteredVariant; import uk.ac.ebi.eva.accession.core.model.IClusteredVariant; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpClusteredVariantEntity; @@ -48,6 +46,7 @@ import uk.ac.ebi.eva.accession.core.model.eva.ClusteredVariantEntity; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; import uk.ac.ebi.eva.accession.core.summary.ClusteredVariantSummaryFunction; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.commons.core.models.pipeline.Variant; import java.io.IOException; @@ -61,48 +60,35 @@ import java.util.function.Function; import java.util.stream.Collectors; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.DUPLICATE_RS_ACC_QC_STEP; import static uk.ac.ebi.eva.accession.clustering.test.configuration.BatchTestConfiguration.JOB_LAUNCHER_DUPLICATE_RS_ACC_QC_JOB; -@RunWith(SpringRunner.class) -@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class}) +@ExtendWith(SpringExtension.class) +@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class, BatchJobRepositoryTestConfiguration.class}) @TestPropertySource("classpath:duplicate-rs-acc-qc-test.properties") -public class DuplicateRSAccQCStepConfigurationTest { - - private static final String TEST_DB = "test-db"; +public class DuplicateRSAccQCStepConfigurationTest extends MongoTestContainerHelper { private static final String duplicateRsAccFile = "src/test/resources/duplicateRSAcc.csv"; @Autowired @Qualifier(JOB_LAUNCHER_DUPLICATE_RS_ACC_QC_JOB) private JobLauncherTestUtils jobLauncherTestUtils; - @Autowired - private MongoClient mongoClient; - @Autowired private MongoTemplate mongoTemplate; - //Required by nosql-unit - @Autowired - private ApplicationContext applicationContext; - private final Gson gson = new GsonBuilder().create(); - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName(TEST_DB).build()); - - @Before + @BeforeEach public void setUp() throws IOException { - this.mongoClient.dropDatabase(TEST_DB); + mongoTemplate.getDb().drop(); Files.deleteIfExists(Paths.get(duplicateRsAccFile)); } - @After + @AfterEach public void tearDown() throws IOException { - this.mongoClient.dropDatabase(TEST_DB); + mongoTemplate.getDb().drop(); Files.deleteIfExists(Paths.get(duplicateRsAccFile)); } @@ -303,7 +289,31 @@ public void assertDuplicateRSAccFileContains(Set duplicateCVEAccessions, String expectedJsonData = gson.toJson(new DuplicateRSAccQCResult(cveAcc, mapOfCveAccAndCveEntities.get(cveAcc), mapOfSveGroupByCveAccAndThenByAsmLoc.get(cveAcc))); - assertEquals(expectedJsonData, fileJsonData); + assertEquals(normalizeNano(expectedJsonData), normalizeNano(fileJsonData)); + } + } + + private String normalizeNano(String json) { + JsonElement element = JsonParser.parseString(json); + removeNano(element); + return element.toString(); + } + + private void removeNano(JsonElement element) { + if (element.isJsonObject()) { + JsonObject obj = element.getAsJsonObject(); + + if (obj.has("nano")) { + obj.addProperty("nano", 0); + } + + for (Map.Entry entry : obj.entrySet()) { + removeNano(entry.getValue()); + } + } else if (element.isJsonArray()) { + for (JsonElement child : element.getAsJsonArray()) { + removeNano(child); + } } } diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/qc/NewClusteredVariantsQCJobConfigurationTest.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/qc/NewClusteredVariantsQCJobConfigurationTest.java index b50197932..91b020eb3 100644 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/qc/NewClusteredVariantsQCJobConfigurationTest.java +++ b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/qc/NewClusteredVariantsQCJobConfigurationTest.java @@ -15,54 +15,61 @@ */ package uk.ac.ebi.eva.accession.clustering.configuration.batch.jobs.qc; -import com.lordofthejars.nosqlunit.annotation.UsingDataSet; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.batch.core.BatchStatus; import org.springframework.batch.core.JobExecution; import org.springframework.batch.core.StepExecution; import org.springframework.batch.test.JobLauncherTestUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.context.ApplicationContext; +import org.springframework.core.io.ResourceLoader; +import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; - +import org.springframework.test.context.junit.jupiter.SpringExtension; +import uk.ac.ebi.eva.accession.clustering.test.configuration.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.accession.clustering.test.configuration.BatchTestConfiguration; -import uk.ac.ebi.eva.accession.clustering.test.rule.FixSpringMongoDbRule; +import uk.ac.ebi.eva.accession.clustering.test.configuration.MongoTestConfiguration; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; +import uk.ac.ebi.eva.accession.core.utils.MongoTestDataLoader; import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.stream.Collectors; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import static uk.ac.ebi.eva.accession.clustering.configuration.batch.jobs.qc.NewClusteredVariantsQCJobConfiguration.REPORT_MISSING_CVE_STEP; import static uk.ac.ebi.eva.accession.clustering.test.configuration.BatchTestConfiguration.JOB_LAUNCHER_NEW_CLUSTERED_VARIANTS_QC; -@RunWith(SpringRunner.class) -@ContextConfiguration(classes = {BatchTestConfiguration.class}) +@ExtendWith(SpringExtension.class) +@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class, BatchJobRepositoryTestConfiguration.class}) @TestPropertySource("classpath:clustering-qc-test.properties") -@UsingDataSet(locations = {"/test-data/clusteredVariantEntityQC.json"}) -public class NewClusteredVariantsQCJobConfigurationTest { - - private static final String TEST_DB = "test-db"; +public class NewClusteredVariantsQCJobConfigurationTest extends MongoTestContainerHelper { @Autowired - @Qualifier(JOB_LAUNCHER_NEW_CLUSTERED_VARIANTS_QC) - private JobLauncherTestUtils jobLauncherTestUtilsNewClusteredVariantsQC; + private MongoTemplate mongoTemplate; - //Required by nosql-unit @Autowired - private ApplicationContext applicationContext; + private ResourceLoader resourceLoader; + + @BeforeEach + public void setUp() { + mongoTemplate.getDb().drop(); + new MongoTestDataLoader(mongoTemplate, resourceLoader).load("/test-data/clusteredVariantEntityQC.json"); + } - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName(TEST_DB).build()); + @AfterEach + public void tearDown() { + mongoTemplate.getDb().drop(); + } + + @Autowired + @Qualifier(JOB_LAUNCHER_NEW_CLUSTERED_VARIANTS_QC) + private JobLauncherTestUtils jobLauncherTestUtilsNewClusteredVariantsQC; @Test public void newClusteredVariantsQCJob() throws Exception { diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/steps/ClusteringVariantStepConfigurationTest.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/steps/ClusteringVariantStepConfigurationTest.java index 28d17ad53..2aa9ddbe5 100644 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/steps/ClusteringVariantStepConfigurationTest.java +++ b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/steps/ClusteringVariantStepConfigurationTest.java @@ -15,38 +15,37 @@ */ package uk.ac.ebi.eva.accession.clustering.configuration.batch.steps; -import com.lordofthejars.nosqlunit.annotation.UsingDataSet; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; import com.mongodb.client.FindIterable; import com.mongodb.client.MongoCollection; import org.bson.Document; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.batch.core.BatchStatus; import org.springframework.batch.core.JobExecution; import org.springframework.batch.core.StepExecution; import org.springframework.batch.test.JobLauncherTestUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.context.ApplicationContext; +import org.springframework.core.io.ResourceLoader; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.http.HttpMethod; import org.springframework.http.HttpStatus; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import org.springframework.test.web.client.ExpectedCount; import org.springframework.test.web.client.MockRestServiceServer; import org.springframework.web.client.RestTemplate; import uk.ac.ebi.ampt2d.commons.accession.core.models.EventType; +import uk.ac.ebi.eva.accession.clustering.test.configuration.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.accession.clustering.test.configuration.BatchTestConfiguration; -import uk.ac.ebi.eva.accession.clustering.test.rule.FixSpringMongoDbRule; +import uk.ac.ebi.eva.accession.clustering.test.configuration.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantOperationEntity; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; +import uk.ac.ebi.eva.accession.core.utils.MongoTestDataLoader; import uk.ac.ebi.eva.metrics.count.CountServiceParameters; import java.net.URI; @@ -55,8 +54,8 @@ import java.util.Collections; import java.util.List; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.springframework.test.web.client.match.MockRestRequestMatchers.method; import static org.springframework.test.web.client.match.MockRestRequestMatchers.requestTo; import static org.springframework.test.web.client.response.MockRestResponseCreators.withStatus; @@ -67,14 +66,11 @@ import static uk.ac.ebi.eva.accession.clustering.test.configuration.BatchTestConfiguration.JOB_LAUNCHER_FROM_MONGO_ONLY_FIRST_STEP; import static uk.ac.ebi.eva.accession.clustering.test.configuration.BatchTestConfiguration.JOB_LAUNCHER_STUDY_FROM_MONGO; -@RunWith(SpringRunner.class) -@ContextConfiguration(classes = {BatchTestConfiguration.class}) +@ExtendWith(SpringExtension.class) +@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class, + BatchJobRepositoryTestConfiguration.class}) @TestPropertySource("classpath:clustering-issuance-test.properties") -@UsingDataSet(locations = {"/test-data/submittedVariantEntity.json"}) -public class ClusteringVariantStepConfigurationTest { - - private static final String TEST_DB = "test-db"; - +public class ClusteringVariantStepConfigurationTest extends MongoTestContainerHelper { private static final String CLUSTERED_VARIANT_COLLECTION = "clusteredVariantEntity"; private static final String SUBMITTED_VARIANT_COLLECTION = "submittedVariantEntity"; @@ -94,6 +90,9 @@ public class ClusteringVariantStepConfigurationTest { @Autowired private MongoTemplate mongoTemplate; + @Autowired + private ResourceLoader resourceLoader; + private MockRestServiceServer mockServer; @Autowired @@ -105,23 +104,17 @@ public class ClusteringVariantStepConfigurationTest { private final String URL_PATH_SAVE_COUNT = "/v1/bulk/count"; - //Required by nosql-unit - @Autowired - private ApplicationContext applicationContext; - - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName(TEST_DB).build()); - - @Before + @BeforeEach public void init() throws Exception { + mongoTemplate.getDb().drop(); + mockServer = MockRestServiceServer.createServer(restTemplate); mockServer.expect(ExpectedCount.manyTimes(), requestTo(new URI(countServiceParameters.getUrl() + URL_PATH_SAVE_COUNT))) .andExpect(method(HttpMethod.POST)) .andRespond(withStatus(HttpStatus.OK)); } - @After + @AfterEach public void tearDown() { mongoTemplate.getDb().drop(); } @@ -129,6 +122,8 @@ public void tearDown() { @Test @DirtiesContext public void nonClusteredVariantStepFromMongo() { + new MongoTestDataLoader(mongoTemplate, resourceLoader).load("/test-data/submittedVariantEntity.json"); + assertEquals(5, mongoTemplate.getCollection(SUBMITTED_VARIANT_COLLECTION).countDocuments()); assertTrue(allSubmittedVariantsNotClustered()); @@ -143,8 +138,9 @@ public void nonClusteredVariantStepFromMongo() { @Test @DirtiesContext - @UsingDataSet(locations = {"/test-data/submittedVariantEntityStudyReader.json"}) public void clusteredVariantStepStudyFromMongo() { + new MongoTestDataLoader(mongoTemplate, resourceLoader).load("/test-data/submittedVariantEntityStudyReader.json"); + assertEquals(5, mongoTemplate.getCollection(SUBMITTED_VARIANT_COLLECTION).countDocuments()); assertEquals(2, getSubmittedVariantsWithRSOrBackPropRS()); @@ -158,9 +154,11 @@ public void clusteredVariantStepStudyFromMongo() { @Test @DirtiesContext - @UsingDataSet(locations = {"/test-data/submittedVariantEntityMongoReader.json", - "/test-data/clusteredVariantEntityMongoReader.json"}) public void clusteredVariantStepFromMongo() throws Exception { + MongoTestDataLoader mongoTestDataLoader = new MongoTestDataLoader(mongoTemplate, resourceLoader); + mongoTestDataLoader.load("/test-data/submittedVariantEntityMongoReader.json"); + mongoTestDataLoader.load("/test-data/clusteredVariantEntityMongoReader.json"); + assertEquals(6, mongoTemplate.getCollection(SUBMITTED_VARIANT_COLLECTION).countDocuments()); assertEquals(1, getSubmittedVariantsWithRSOrBackPropRS()); @@ -168,7 +166,7 @@ public void clusteredVariantStepFromMongo() throws Exception { // the following cannot be executed directly, therefore we launch the entire job and ensure that the // CLUSTERING_CLUSTERED_VARIANTS_FROM_MONGO_STEP step was completed //JobExecution jobExecution = jobLauncherTestUtilsFromMongo.launchStep( - //CLUSTERING_CLUSTERED_VARIANTS_FROM_MONGO_STEP); + //CLUSTERING_CLUSTERED_VARIANTS_FROM_MONGO_STEP); JobExecution jobExecution = jobLauncherTestUtilsFromMongoOnlyFirstStep.launchJob(); assertEquals(BatchStatus.COMPLETED, jobExecution.getStatus()); assertTrue(jobExecution.getStepExecutions().stream().map(StepExecution::getStepName) diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/runner/ClusteringCommandLineRunnerTest.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/runner/ClusteringCommandLineRunnerTest.java index dfdd4ddba..7206b6cf1 100644 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/runner/ClusteringCommandLineRunnerTest.java +++ b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/runner/ClusteringCommandLineRunnerTest.java @@ -16,22 +16,19 @@ */ package uk.ac.ebi.eva.accession.clustering.runner; -import com.lordofthejars.nosqlunit.annotation.UsingDataSet; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mockito; import org.springframework.batch.core.JobExecution; import org.springframework.batch.core.JobExecutionException; import org.springframework.batch.core.JobInstance; import org.springframework.batch.core.explore.JobExplorer; import org.springframework.batch.core.repository.JobRepository; +import org.springframework.batch.item.Chunk; import org.springframework.batch.item.ExecutionContext; import org.springframework.batch.item.ItemWriter; import org.springframework.batch.test.JobLauncherTestUtils; @@ -39,7 +36,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.test.mock.mockito.MockBean; -import org.springframework.context.ApplicationContext; +import org.springframework.core.io.ResourceLoader; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.data.mongodb.core.query.Query; import org.springframework.http.HttpMethod; @@ -47,7 +44,7 @@ import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import org.springframework.test.web.client.ExpectedCount; import org.springframework.test.web.client.MockRestServiceServer; import org.springframework.web.client.RestTemplate; @@ -65,8 +62,9 @@ import uk.ac.ebi.eva.accession.clustering.configuration.batch.io.RSMergeAndSplitCandidatesReaderConfiguration; import uk.ac.ebi.eva.accession.clustering.parameters.InputParameters; import uk.ac.ebi.eva.accession.clustering.test.DatabaseState; +import uk.ac.ebi.eva.accession.clustering.test.configuration.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.accession.clustering.test.configuration.BatchTestConfiguration; -import uk.ac.ebi.eva.accession.clustering.test.rule.FixSpringMongoDbRule; +import uk.ac.ebi.eva.accession.clustering.test.configuration.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.model.ClusteredVariant; import uk.ac.ebi.eva.accession.core.model.IClusteredVariant; import uk.ac.ebi.eva.accession.core.model.ISubmittedVariant; @@ -84,12 +82,13 @@ import uk.ac.ebi.eva.accession.core.service.nonhuman.SubmittedVariantAccessioningService; import uk.ac.ebi.eva.accession.core.summary.ClusteredVariantSummaryFunction; import uk.ac.ebi.eva.accession.core.summary.SubmittedVariantSummaryFunction; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; +import uk.ac.ebi.eva.accession.core.utils.MongoTestDataLoader; import uk.ac.ebi.eva.commons.core.models.VariantClassifier; import uk.ac.ebi.eva.commons.core.models.VariantType; import uk.ac.ebi.eva.commons.mongodb.readers.MongoDbCursorItemReader; import uk.ac.ebi.eva.metrics.count.CountServiceParameters; -import javax.sql.DataSource; import java.io.File; import java.net.URI; import java.util.ArrayList; @@ -101,26 +100,36 @@ import java.util.function.Function; import java.util.stream.Collectors; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.springframework.data.mongodb.core.query.Criteria.where; import static org.springframework.data.mongodb.core.query.Query.query; import static org.springframework.test.web.client.match.MockRestRequestMatchers.method; import static org.springframework.test.web.client.match.MockRestRequestMatchers.requestTo; import static org.springframework.test.web.client.response.MockRestResponseCreators.withStatus; -import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.*; -import static uk.ac.ebi.eva.accession.clustering.test.configuration.BatchTestConfiguration.JOB_LAUNCHER_FROM_MONGO; +import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.BACK_PROPAGATE_NEW_RS_STEP; +import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.BACK_PROPAGATE_SPLIT_OR_MERGED_RS_STEP; +import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.CLEAR_RS_MERGE_AND_SPLIT_CANDIDATES_STEP; +import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.CLUSTERED_CLUSTERING_WRITER; +import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.CLUSTERING_CLUSTERED_VARIANTS_FROM_MONGO_STEP; +import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.CLUSTERING_FROM_MONGO_JOB; +import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.CLUSTERING_NON_CLUSTERED_VARIANTS_FROM_MONGO_STEP; +import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.CLUSTER_UNCLUSTERED_VARIANTS_JOB; +import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.PROCESS_REMAPPED_VARIANTS_WITH_RS_JOB; +import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.PROCESS_RS_MERGE_CANDIDATES_STEP; +import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.PROCESS_RS_SPLIT_CANDIDATES_STEP; +import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.RS_MERGE_WRITER; +import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.RS_SPLIT_WRITER; import static uk.ac.ebi.eva.accession.clustering.test.configuration.BatchTestConfiguration.JOB_LAUNCHER_FROM_MONGO_ONLY_FIRST_STEP; -@RunWith(SpringRunner.class) -@ContextConfiguration(classes = {BatchTestConfiguration.class}) +@ExtendWith(SpringExtension.class) +@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class, + BatchJobRepositoryTestConfiguration.class}) @TestPropertySource("classpath:clustering-pipeline-test.properties") -public class ClusteringCommandLineRunnerTest { +public class ClusteringCommandLineRunnerTest extends MongoTestContainerHelper { private static String TEST_APPLICATION_INSTANCE_ID = "test-application-instance-id"; - private static final String TEST_DB = "test-db"; - private static final int TAXONOMY = 60711; private static String ASM1; @@ -156,18 +165,14 @@ public class ClusteringCommandLineRunnerTest { @Autowired private JobExplorer jobExplorer; - @Autowired - private DataSource datasource; - @Autowired private ClusteringCommandLineRunner runner; @Autowired private MongoTemplate mongoTemplate; - //Required by nosql-unit @Autowired - private ApplicationContext applicationContext; + private ResourceLoader resourceLoader; @Autowired private MonotonicAccessionGenerator clusteredVariantAccessionGenerator; @@ -182,25 +187,10 @@ public class ClusteringCommandLineRunnerTest { @Qualifier(CLUSTERED_CLUSTERING_WRITER) private ClusteringWriter clusteringWriter; - @Autowired - @Qualifier(JOB_LAUNCHER_FROM_MONGO) - private JobLauncherTestUtils jobLauncherTestUtilsFromMongo; - @Autowired @Qualifier(JOB_LAUNCHER_FROM_MONGO_ONLY_FIRST_STEP) private JobLauncherTestUtils jobLauncherTestUtilsFromMongoOnlyFirstStep; - // Current clustering sequence is: - // generate merge split candidates from clustered variants -> perform merge - // -> perform split -> cluster new variants - @Autowired - @Qualifier(CLUSTERED_CLUSTERING_WRITER) - private ClusteringWriter clusteringWriterPreMergeAndSplit; - - @Autowired - @Qualifier(NON_CLUSTERED_CLUSTERING_WRITER) - private ClusteringWriter clusteringWriterPostMergeAndSplit; - private MongoDbCursorItemReader rsMergeCandidatesReader; private MongoDbCursorItemReader rsSplitCandidatesReader; @@ -216,11 +206,6 @@ public class ClusteringCommandLineRunnerTest { @MockBean private JobExecution jobExecution; - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName(TEST_DB).build()); - - private JobRepositoryTestUtils jobRepositoryTestUtils; private static String originalRemappedFrom; @@ -269,17 +254,17 @@ public String getHash() { private final String URL_PATH_SAVE_COUNT = "/v1/bulk/count"; - @BeforeClass + @BeforeAll public static void initializeTempFile() throws Exception { tempVcfInputFileToTestFailingJobs = File.createTempFile("resumeFailingJob", ".vcf.gz"); } - @AfterClass - public static void deleteTempFile() throws Exception { + @AfterAll + public static void deleteTempFile() { tempVcfInputFileToTestFailingJobs.delete(); } - @Before + @BeforeEach public void setUp() throws Exception { if (!originalInputParametersCaptured) { originalRemappedFrom = inputParameters.getRemappedFrom(); @@ -287,7 +272,7 @@ public void setUp() throws Exception { } ASM1 = inputParameters.getRemappedFrom(); ASM2 = inputParameters.getAssemblyAccession(); - jobRepositoryTestUtils = new JobRepositoryTestUtils(jobRepository, datasource); + jobRepositoryTestUtils = new JobRepositoryTestUtils(jobRepository); jobRepositoryTestUtils.removeJobExecutions(); inputParameters.setForceRestart(false); inputParameters.setRemappedFrom(originalRemappedFrom); @@ -302,7 +287,7 @@ public void setUp() throws Exception { rsSplitWriter.setJobExecution(jobExecution); } - @After + @AfterEach public void tearDown() { jobRepositoryTestUtils.removeJobExecutions(); inputParameters.setForceRestart(false); @@ -310,10 +295,11 @@ public void tearDown() { } @Test - @UsingDataSet(locations = {"/test-data/submittedVariantEntityMongoReader.json"}) @DirtiesContext public void runFullClusteringFromMongoJobWithNoErrors() throws JobExecutionException { - runner.setJobNames(CLUSTERING_FROM_MONGO_JOB); + new MongoTestDataLoader(mongoTemplate, resourceLoader).load("/test-data/submittedVariantEntityMongoReader.json"); + + runner.setJobName(CLUSTERING_FROM_MONGO_JOB); runner.run(); assertEquals(ClusteringCommandLineRunner.EXIT_WITHOUT_ERRORS, runner.getExitCode()); @@ -338,13 +324,14 @@ public void runFullClusteringFromMongoJobWithNoErrors() throws JobExecutionExcep } @Test - @UsingDataSet(locations = {"/test-data/submittedVariantEntityMongoReader.json"}) @DirtiesContext // For a clustering job involving non-remapped variants, only one step i.e., // CLUSTERING_NON_CLUSTERED_VARIANTS_FROM_MONGO_STEP should be executed public void runPartialClusteringFromMongoJobForNonRemappedVariants() throws JobExecutionException { + new MongoTestDataLoader(mongoTemplate, resourceLoader).load("/test-data/submittedVariantEntityMongoReader.json"); + inputParameters.setRemappedFrom(null); - runner.setJobNames(CLUSTERING_FROM_MONGO_JOB); + runner.setJobName(CLUSTERING_FROM_MONGO_JOB); runner.run(); assertEquals(ClusteringCommandLineRunner.EXIT_WITHOUT_ERRORS, runner.getExitCode()); @@ -368,10 +355,11 @@ public void runPartialClusteringFromMongoJobForNonRemappedVariants() throws JobE } @Test - @UsingDataSet(locations = {"/test-data/submittedVariantEntityMongoReader.json"}) @DirtiesContext public void runProcessRemappedRSJobWithNoErrors() throws JobExecutionException { - runner.setJobNames(PROCESS_REMAPPED_VARIANTS_WITH_RS_JOB); + new MongoTestDataLoader(mongoTemplate, resourceLoader).load("/test-data/submittedVariantEntityMongoReader.json"); + + runner.setJobName(PROCESS_REMAPPED_VARIANTS_WITH_RS_JOB); runner.run(); assertEquals(ClusteringCommandLineRunner.EXIT_WITHOUT_ERRORS, runner.getExitCode()); } @@ -395,7 +383,7 @@ public void runProcessRemappedRSJobMultipleTimes() throws JobExecutionException SubmittedVariantEntity ss1 = createSS(1L, 1L, rsLocus1, "A", "T", true); SubmittedVariantEntity ss2 = createSS(2L, 2L, rsLocus1, "A", "G", true); SubmittedVariantEntity ss3 = createSS(3L, 3L, rsLocus2, "T", "", true); - runner.setJobNames(PROCESS_REMAPPED_VARIANTS_WITH_RS_JOB); + runner.setJobName(PROCESS_REMAPPED_VARIANTS_WITH_RS_JOB); runner.run(); assertEquals(ClusteringCommandLineRunner.EXIT_WITHOUT_ERRORS, runner.getExitCode()); mergeOperations.add(Arrays.asList(ss1, ss2)); @@ -405,7 +393,7 @@ public void runProcessRemappedRSJobMultipleTimes() throws JobExecutionException // Second run SubmittedVariantEntity ss4 = createSS(4L, 3L, rsLocus3, "", "T", true); SubmittedVariantEntity ss5 = createSS(5L, 4L, rsLocus4, "A", "G", true); - runner.setJobNames(PROCESS_REMAPPED_VARIANTS_WITH_RS_JOB); + runner.setJobName(PROCESS_REMAPPED_VARIANTS_WITH_RS_JOB); runner.run(); assertEquals(ClusteringCommandLineRunner.EXIT_WITHOUT_ERRORS, runner.getExitCode()); splitOperations.add(Arrays.asList(ss3, ss4)); @@ -414,7 +402,7 @@ public void runProcessRemappedRSJobMultipleTimes() throws JobExecutionException // Third run SubmittedVariantEntity ss6 = createSS(6L, 5L, rsLocus4, "G", "A", true); - runner.setJobNames(PROCESS_REMAPPED_VARIANTS_WITH_RS_JOB); + runner.setJobName(PROCESS_REMAPPED_VARIANTS_WITH_RS_JOB); runner.run(); assertEquals(ClusteringCommandLineRunner.EXIT_WITHOUT_ERRORS, runner.getExitCode()); mergeOperations.add(Arrays.asList(ss5, ss6)); @@ -422,7 +410,7 @@ public void runProcessRemappedRSJobMultipleTimes() throws JobExecutionException // Fourth run createSS(7L, 6L, rsLocus5, "", "A", true); - runner.setJobNames(PROCESS_REMAPPED_VARIANTS_WITH_RS_JOB); + runner.setJobName(PROCESS_REMAPPED_VARIANTS_WITH_RS_JOB); runner.run(); assertEquals(ClusteringCommandLineRunner.EXIT_WITHOUT_ERRORS, runner.getExitCode()); insertedRSHashes.add(rsLocus5.getHash()); @@ -495,10 +483,11 @@ private boolean areAllSSPresentInOperation(List expected } @Test - @UsingDataSet(locations = {"/test-data/submittedVariantEntityMongoReader.json"}) @DirtiesContext public void runClusterUnclusteredVariantsJobWithNoErrors() throws JobExecutionException { - runner.setJobNames(CLUSTER_UNCLUSTERED_VARIANTS_JOB); + new MongoTestDataLoader(mongoTemplate, resourceLoader).load("/test-data/submittedVariantEntityMongoReader.json"); + + runner.setJobName(CLUSTER_UNCLUSTERED_VARIANTS_JOB); runner.run(); assertEquals(ClusteringCommandLineRunner.EXIT_WITHOUT_ERRORS, runner.getExitCode()); } @@ -511,7 +500,7 @@ public void runClusterUnclusteredVariantsJobWithNoErrors() throws JobExecutionEx public void runBackPropagateRSJobWithNoErrors() throws JobExecutionException, AccessionCouldNotBeGeneratedException { setupRSAndSSInNewAndRemappedAssemblies(); - runner.setJobNames(CLUSTERING_FROM_MONGO_JOB); + runner.setJobName(CLUSTERING_FROM_MONGO_JOB); runner.run(); assertEquals(ClusteringCommandLineRunner.EXIT_WITHOUT_ERRORS, runner.getExitCode()); @@ -549,7 +538,7 @@ public void runClusteringMongoJobWithOverlappingMergesAndSplits() throws JobExec SubmittedVariantEntity ss2_another = createSS(2L, rs3.getAccession(), rsLocus2, "A", "", true); SubmittedVariantEntity ss3 = createSS(3L, rs2.getAccession(), rsLocus2, "C", "", true); - runner.setJobNames(CLUSTERING_FROM_MONGO_JOB); + runner.setJobName(CLUSTERING_FROM_MONGO_JOB); runner.run(); assertEquals(ClusteringCommandLineRunner.EXIT_WITHOUT_ERRORS, runner.getExitCode()); @@ -577,7 +566,7 @@ Initial state and expected results here (marked in green): https://docs.google.c setupRSAndSS(); createEVASS8InASM1WithUnassignedRS(); - runner.setJobNames(CLUSTERING_FROM_MONGO_JOB); + runner.setJobName(CLUSTERING_FROM_MONGO_JOB); runner.run(); assertEquals(ClusteringCommandLineRunner.EXIT_WITHOUT_ERRORS, runner.getExitCode()); @@ -910,7 +899,7 @@ private ClusteredVariantEntity createRS(Long rsAccession, RSLocus rsLocus, boole @Test @DirtiesContext public void runJobWithNoErrors() throws JobExecutionException { - runner.setJobNames(CLUSTERING_FROM_MONGO_JOB); + runner.setJobName(CLUSTERING_FROM_MONGO_JOB); runner.run(); assertEquals(ClusteringCommandLineRunner.EXIT_WITHOUT_ERRORS, runner.getExitCode()); } @@ -918,7 +907,7 @@ public void runJobWithNoErrors() throws JobExecutionException { @Test @DirtiesContext public void runJobWithNoName() throws JobExecutionException { - runner.setJobNames(null); + runner.setJobName(null); runner.run(); assertEquals(ClusteringCommandLineRunner.EXIT_WITH_ERRORS, runner.getExitCode()); } @@ -926,7 +915,7 @@ public void runJobWithNoName() throws JobExecutionException { @Test @DirtiesContext public void runNonExistentJob() throws JobExecutionException { - runner.setJobNames("NOT_EXISTENT_JOB"); + runner.setJobName("NOT_EXISTENT_JOB"); runner.run(); assertEquals(ClusteringCommandLineRunner.EXIT_WITH_ERRORS, runner.getExitCode()); } @@ -934,7 +923,7 @@ public void runNonExistentJob() throws JobExecutionException { @Test @DirtiesContext public void restartCompletedJobThatIsAlreadyInTheRepository() throws Exception { - runner.setJobNames(CLUSTERING_FROM_MONGO_JOB); + runner.setJobName(CLUSTERING_FROM_MONGO_JOB); runner.run(); assertEquals(ClusteringCommandLineRunner.EXIT_WITHOUT_ERRORS, runner.getExitCode()); @@ -947,7 +936,7 @@ public void restartCompletedJobThatIsAlreadyInTheRepository() throws Exception { @Test @DirtiesContext public void forceRestartButNoJobInTheRepository() throws Exception { - runner.setJobNames(CLUSTERING_FROM_MONGO_JOB); + runner.setJobName(CLUSTERING_FROM_MONGO_JOB); inputParameters.setForceRestart(true); assertEquals(Collections.EMPTY_LIST, jobExplorer.getJobNames()); runner.run(); @@ -990,13 +979,13 @@ private void runSplitAndMergeIdentificationAndHandlingSteps() while ((tempSVO = rsMergeCandidatesReader.read()) != null) { mergeCandidates.add(tempSVO); } - rsMergeWriter.write(mergeCandidates); + rsMergeWriter.write(new Chunk<>(mergeCandidates)); rsSplitCandidatesReader = new RSMergeAndSplitCandidatesReaderConfiguration() .rsSplitCandidatesReader(this.mongoTemplate, this.inputParameters); rsSplitCandidatesReader.open(new ExecutionContext()); while ((tempSVO = rsSplitCandidatesReader.read()) != null) { splitCandidates.add(tempSVO); } - rsSplitWriter.write(splitCandidates); + rsSplitWriter.write(new Chunk<>(splitCandidates)); } } \ No newline at end of file diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/test/VariantAssertions.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/test/VariantAssertions.java index 5793cece4..e019219f0 100644 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/test/VariantAssertions.java +++ b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/test/VariantAssertions.java @@ -23,7 +23,7 @@ import java.util.TreeSet; import java.util.stream.Collectors; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; public class VariantAssertions { diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/test/configuration/BatchJobRepositoryTestConfiguration.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/test/configuration/BatchJobRepositoryTestConfiguration.java new file mode 100644 index 000000000..6cebe2481 --- /dev/null +++ b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/test/configuration/BatchJobRepositoryTestConfiguration.java @@ -0,0 +1,22 @@ +package uk.ac.ebi.eva.accession.clustering.test.configuration; + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Primary; +import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseBuilder; +import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseType; + +import javax.sql.DataSource; + +@Configuration +public class BatchJobRepositoryTestConfiguration { + + @Bean + @Primary + public DataSource dataSource() { + return new EmbeddedDatabaseBuilder() + .setType(EmbeddedDatabaseType.HSQL) + .addScript("org/springframework/batch/core/schema-hsqldb.sql") + .build(); + } +} \ No newline at end of file diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/test/configuration/BatchTestConfiguration.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/test/configuration/BatchTestConfiguration.java index 2fc8d7ed1..cc5eb69f8 100644 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/test/configuration/BatchTestConfiguration.java +++ b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/test/configuration/BatchTestConfiguration.java @@ -18,15 +18,15 @@ package uk.ac.ebi.eva.accession.clustering.test.configuration; import org.springframework.batch.core.Job; +import org.springframework.batch.core.launch.JobLauncher; +import org.springframework.batch.core.repository.JobRepository; import org.springframework.batch.test.JobLauncherTestUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; -import org.springframework.boot.autoconfigure.batch.BatchProperties; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Import; import org.springframework.core.io.ResourceLoader; -import org.springframework.transaction.PlatformTransactionManager; import uk.ac.ebi.eva.accession.clustering.configuration.batch.io.BackPropagatedRSWriterConfiguration; import uk.ac.ebi.eva.accession.clustering.configuration.batch.io.ClusteringMongoReaderConfiguration; import uk.ac.ebi.eva.accession.clustering.configuration.batch.io.ClusteringWriterConfiguration; @@ -57,8 +57,6 @@ import uk.ac.ebi.eva.accession.clustering.runner.ClusteringCommandLineRunner; import uk.ac.ebi.eva.commons.batch.job.JobExecutionApplicationListener; -import javax.sql.DataSource; - import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.CLUSTERING_FROM_MONGO_JOB; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.DUPLICATE_RS_ACC_QC_JOB; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.PROCESS_REMAPPED_VARIANTS_WITH_RS_JOB; @@ -110,76 +108,63 @@ public class BatchTestConfiguration { public static final String JOB_LAUNCHER_DUPLICATE_RS_ACC_QC_JOB = "JOB_LAUNCHER_DUPLICATE_RS_ACC_QC_JOB"; - @Autowired - private BatchProperties properties; - @Autowired private ResourceLoader resourceLoader; - @Autowired - private DataSource dataSource; - - @Autowired - private PlatformTransactionManager platformTransactionManager; - @Bean(JOB_LAUNCHER_FROM_MONGO) - public JobLauncherTestUtils jobLauncherTestUtilsFromMongo() { - - return new JobLauncherTestUtils() { - @Override - @Autowired - public void setJob(@Qualifier(CLUSTERING_FROM_MONGO_JOB) Job job) { - super.setJob(job); - } - }; + public JobLauncherTestUtils jobLauncherTestUtilsFromMongo(JobLauncher jobLauncher, JobRepository jobRepository, + @Qualifier(CLUSTERING_FROM_MONGO_JOB) Job job) { + JobLauncherTestUtils utils = new JobLauncherTestUtils(); + utils.setJobLauncher(jobLauncher); + utils.setJobRepository(jobRepository); + utils.setJob(job); + return utils; } @Bean(JOB_LAUNCHER_STUDY_FROM_MONGO) - public JobLauncherTestUtils jobLauncherTestUtilsStudyFromMongo() { - - return new JobLauncherTestUtils() { - @Override - @Autowired - public void setJob(@Qualifier(STUDY_CLUSTERING_JOB) Job job) { - super.setJob(job); - } - }; + public JobLauncherTestUtils jobLauncherTestUtilsStudyFromMongo(JobLauncher jobLauncher, JobRepository jobRepository, + @Qualifier(STUDY_CLUSTERING_JOB) Job job) { + + JobLauncherTestUtils utils = new JobLauncherTestUtils(); + utils.setJobLauncher(jobLauncher); + utils.setJobRepository(jobRepository); + utils.setJob(job); + return utils; } @Bean(JOB_LAUNCHER_RS_ACCESSION_RECOVERY) - public JobLauncherTestUtils jobLauncherTestUtilsRSAccessionRecoveryJob() { - - return new JobLauncherTestUtils() { - @Override - @Autowired - public void setJob(@Qualifier(RS_ACCESSION_RECOVERY_JOB) Job job) { - super.setJob(job); - } - }; + public JobLauncherTestUtils jobLauncherTestUtilsRSAccessionRecoveryJob(JobLauncher jobLauncher, JobRepository jobRepository, + @Qualifier(RS_ACCESSION_RECOVERY_JOB) Job job) { + + JobLauncherTestUtils utils = new JobLauncherTestUtils(); + utils.setJobLauncher(jobLauncher); + utils.setJobRepository(jobRepository); + utils.setJob(job); + return utils; } @Bean(JOB_LAUNCHER_NEW_CLUSTERED_VARIANTS_QC) - public JobLauncherTestUtils jobLauncherTestUtilsNewClusteredVariants() { - - return new JobLauncherTestUtils() { - @Override - @Autowired - public void setJob(@Qualifier(NEW_CLUSTERED_VARIANTS_QC_JOB) Job job) { - super.setJob(job); - } - }; + public JobLauncherTestUtils jobLauncherTestUtilsNewClusteredVariants(JobLauncher jobLauncher, JobRepository jobRepository, + @Qualifier(NEW_CLUSTERED_VARIANTS_QC_JOB) Job job) { + + JobLauncherTestUtils utils = new JobLauncherTestUtils(); + utils.setJobLauncher(jobLauncher); + utils.setJobRepository(jobRepository); + utils.setJob(job); + return utils; + } @Bean(JOB_LAUNCHER_FROM_MONGO_ONLY_FIRST_STEP) - public JobLauncherTestUtils jobLauncherTestUtilsFromMongoOnlyFirstStep() { - - return new JobLauncherTestUtils() { - @Override - @Autowired - public void setJob(@Qualifier(PROCESS_REMAPPED_VARIANTS_WITH_RS_JOB) Job job) { - super.setJob(job); - } - }; + public JobLauncherTestUtils jobLauncherTestUtilsFromMongoOnlyFirstStep(JobLauncher jobLauncher, JobRepository jobRepository, + @Qualifier(PROCESS_REMAPPED_VARIANTS_WITH_RS_JOB) Job job) { + + JobLauncherTestUtils utils = new JobLauncherTestUtils(); + utils.setJobLauncher(jobLauncher); + utils.setJobRepository(jobRepository); + utils.setJob(job); + return utils; + } @Bean @@ -188,13 +173,13 @@ public JobExecutionApplicationListener jobExecutionApplicationListener() { } @Bean(JOB_LAUNCHER_DUPLICATE_RS_ACC_QC_JOB) - public JobLauncherTestUtils jobLauncherTestUtilsDuplicateRSAccQCJob() { - return new JobLauncherTestUtils() { - @Override - @Autowired - public void setJob(@Qualifier(DUPLICATE_RS_ACC_QC_JOB) Job job) { - super.setJob(job); - } - }; + public JobLauncherTestUtils jobLauncherTestUtilsDuplicateRSAccQCJob(JobLauncher jobLauncher, JobRepository jobRepository, + @Qualifier(DUPLICATE_RS_ACC_QC_JOB) Job job) { + + JobLauncherTestUtils utils = new JobLauncherTestUtils(); + utils.setJobLauncher(jobLauncher); + utils.setJobRepository(jobRepository); + utils.setJob(job); + return utils; } } diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/test/configuration/MongoTestConfiguration.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/test/configuration/MongoTestConfiguration.java index 6bf441aca..a4a8fc2c0 100644 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/test/configuration/MongoTestConfiguration.java +++ b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/test/configuration/MongoTestConfiguration.java @@ -15,12 +15,30 @@ */ package uk.ac.ebi.eva.accession.clustering.test.configuration; +import com.mongodb.ReadPreference; +import com.mongodb.WriteConcern; +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoClients; +import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.domain.EntityScan; import org.springframework.boot.test.autoconfigure.data.mongo.AutoConfigureDataMongo; +import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Primary; +import org.springframework.data.mongodb.MongoDatabaseFactory; import org.springframework.data.mongodb.config.EnableMongoAuditing; +import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.data.mongodb.core.SimpleMongoClientDatabaseFactory; +import org.springframework.data.mongodb.core.WriteResultChecking; +import org.springframework.data.mongodb.core.convert.DefaultDbRefResolver; +import org.springframework.data.mongodb.core.convert.DefaultMongoTypeMapper; +import org.springframework.data.mongodb.core.convert.MappingMongoConverter; +import org.springframework.data.mongodb.core.convert.MongoCustomConversions; +import org.springframework.data.mongodb.core.mapping.MongoMappingContext; import org.springframework.data.mongodb.repository.config.EnableMongoRepositories; +import java.util.Collections; + @Configuration @EntityScan(basePackages = {"uk.ac.ebi.eva.accession.core.persistence"}) @EnableMongoRepositories(basePackages = { @@ -29,4 +47,56 @@ @EnableMongoAuditing @AutoConfigureDataMongo public class MongoTestConfiguration { -} + + @Value("${spring.data.mongodb.uri}") + private String mongoUri; + + @Value("${spring.data.mongodb.database}") + private String database; + + @Value("${mongodb.read-preference:primary}") + private String readPreference; + + @Bean + @Primary + public MongoClient mongoClient() { + return MongoClients.create(mongoUri); + } + + @Bean("primaryFactory") + @Primary + public MongoDatabaseFactory mongoDbFactory() { + return new SimpleMongoClientDatabaseFactory(mongoClient(), database); + } + + @Bean + @Primary + public MongoCustomConversions mongoCustomConversions() { + return new MongoCustomConversions(Collections.emptyList()); + } + + @Bean + @Primary + public MongoMappingContext mongoMappingContext(MongoCustomConversions conversions) { + MongoMappingContext context = new MongoMappingContext(); + context.setSimpleTypeHolder(conversions.getSimpleTypeHolder()); + context.setAutoIndexCreation(false); + return context; + } + + @Bean + @Primary + public MongoTemplate mongoTemplate(MongoMappingContext mongoMappingContext, + MongoCustomConversions conversions) { + MappingMongoConverter converter = new MappingMongoConverter( + new DefaultDbRefResolver(mongoDbFactory()), mongoMappingContext); + converter.setCustomConversions(conversions); + converter.setTypeMapper(new DefaultMongoTypeMapper(null)); + converter.afterPropertiesSet(); + MongoTemplate mongoTemplate = new MongoTemplate(mongoDbFactory(), converter); + mongoTemplate.setReadPreference(ReadPreference.valueOf(readPreference)); + mongoTemplate.setWriteConcern(WriteConcern.MAJORITY); + mongoTemplate.setWriteResultChecking(WriteResultChecking.EXCEPTION); + return mongoTemplate; + } +} \ No newline at end of file diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/test/configuration/RSAccessionRecoveryTestConfiguration.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/test/configuration/RSAccessionRecoveryTestConfiguration.java index b1bdcf1ab..6cd5d315c 100644 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/test/configuration/RSAccessionRecoveryTestConfiguration.java +++ b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/test/configuration/RSAccessionRecoveryTestConfiguration.java @@ -17,9 +17,9 @@ package uk.ac.ebi.eva.accession.clustering.test.configuration; import org.springframework.batch.core.Job; -import org.springframework.batch.core.configuration.annotation.BatchConfigurer; +import org.springframework.batch.core.launch.JobLauncher; +import org.springframework.batch.core.repository.JobRepository; import org.springframework.batch.test.JobLauncherTestUtils; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.context.annotation.Bean; @@ -29,11 +29,6 @@ import uk.ac.ebi.eva.accession.clustering.configuration.batch.recovery.RSAccessionRecoveryServiceConfiguration; import uk.ac.ebi.eva.accession.clustering.configuration.batch.steps.RSAccessionRecoveryStepConfiguration; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.ClusteredVariantAccessioningConfiguration; -import uk.ac.ebi.eva.commons.batch.configuration.SpringBoot1CompatibilityConfiguration; -import uk.ac.ebi.eva.commons.batch.job.JobExecutionApplicationListener; - -import javax.persistence.EntityManagerFactory; -import javax.sql.DataSource; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.RS_ACCESSION_RECOVERY_JOB; @@ -47,35 +42,14 @@ public class RSAccessionRecoveryTestConfiguration { public static final String JOB_LAUNCHER_RS_ACCESSION_RECOVERY = "JOB_LAUNCHER_RS_ACCESSION_RECOVERY"; - @Bean - public BatchConfigurer configurer(DataSource dataSource, EntityManagerFactory entityManagerFactory) - throws Exception { - return SpringBoot1CompatibilityConfiguration.getSpringBoot1CompatibleBatchConfigurer(dataSource, - entityManagerFactory); - } - - @Bean - public JobLauncherTestUtils jobLauncherTestUtils(BatchConfigurer configurer) throws Exception { - JobLauncherTestUtils jobLauncherTestUtils = new JobLauncherTestUtils(); - jobLauncherTestUtils.setJobLauncher(configurer.getJobLauncher()); - jobLauncherTestUtils.setJobRepository(configurer.getJobRepository()); - return jobLauncherTestUtils; - } - - @Bean - public JobExecutionApplicationListener jobExecutionApplicationListener() { - return new JobExecutionApplicationListener(); - } - @Bean(JOB_LAUNCHER_RS_ACCESSION_RECOVERY) - public JobLauncherTestUtils jobLauncherTestUtilsMonotonicAccessionRecoveryAgent() { - - return new JobLauncherTestUtils() { - @Override - @Autowired - public void setJob(@Qualifier(RS_ACCESSION_RECOVERY_JOB) Job job) { - super.setJob(job); - } - }; + public JobLauncherTestUtils jobLauncherTestUtilsMonotonicAccessionRecoveryAgent(JobLauncher jobLauncher, + JobRepository jobRepository, + @Qualifier(RS_ACCESSION_RECOVERY_JOB) Job job) { + JobLauncherTestUtils utils = new JobLauncherTestUtils(); + utils.setJobLauncher(jobLauncher); + utils.setJobRepository(jobRepository); + utils.setJob(job); + return utils; } } \ No newline at end of file diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/test/rule/FixSpringMongoDbRule.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/test/rule/FixSpringMongoDbRule.java deleted file mode 100644 index 001faf1c6..000000000 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/test/rule/FixSpringMongoDbRule.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * - * * Copyright 2020 EMBL - European Bioinformatics Institute - * * - * * Licensed under the Apache License, Version 2.0 (the "License"); - * * you may not use this file except in compliance with the License. - * * You may obtain a copy of the License at - * * - * * http://www.apache.org/licenses/LICENSE-2.0 - * * - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, - * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * See the License for the specific language governing permissions and - * * limitations under the License. - * - */ -package uk.ac.ebi.eva.accession.clustering.test.rule; - -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfiguration; -import com.lordofthejars.nosqlunit.mongodb.SpringMongoDbRule; - -/** - * Temporary fix until nosql unit rc-6 or final is released - */ -public class FixSpringMongoDbRule extends SpringMongoDbRule { - - public FixSpringMongoDbRule(MongoDbConfiguration mongoDbConfiguration) { - super(mongoDbConfiguration); - } - - public FixSpringMongoDbRule(MongoDbConfiguration mongoDbConfiguration, Object object) { - super(mongoDbConfiguration, object); - } - - @Override - public void close() { - // DO NOT CLOSE the connection (Spring will do it when destroying the context) - } -} diff --git a/eva-accession-clustering/src/test/resources/backpropagation-test.properties b/eva-accession-clustering/src/test/resources/backpropagation-test.properties index 3c24d4c6d..336a50626 100644 --- a/eva-accession-clustering/src/test/resources/backpropagation-test.properties +++ b/eva-accession-clustering/src/test/resources/backpropagation-test.properties @@ -23,9 +23,6 @@ accessioning.monotonic.rs.nextBlockInterval=1000000000 spring.data.mongodb.database=test-db -spring.data.mongodb.host=|eva.mongo.host.test| -spring.data.mongodb.password= -spring.data.mongodb.port=27017 mongodb.read-preference=primary # See https://github.com/spring-projects/spring-boot/wiki/Spring-Boot-2.1-Release-Notes#bean-overriding diff --git a/eva-accession-clustering/src/test/resources/clustering-issuance-test.properties b/eva-accession-clustering/src/test/resources/clustering-issuance-test.properties index 5184a2b48..d39fc667d 100644 --- a/eva-accession-clustering/src/test/resources/clustering-issuance-test.properties +++ b/eva-accession-clustering/src/test/resources/clustering-issuance-test.properties @@ -24,9 +24,6 @@ accessioning.monotonic.rs.nextBlockInterval=1000000000 spring.data.mongodb.database=test-db -spring.data.mongodb.host=|eva.mongo.host.test| -spring.data.mongodb.password= -spring.data.mongodb.port=27017 mongodb.read-preference=primary # See https://github.com/spring-projects/spring-boot/wiki/Spring-Boot-2.1-Release-Notes#bean-overriding diff --git a/eva-accession-clustering/src/test/resources/clustering-pipeline-test.properties b/eva-accession-clustering/src/test/resources/clustering-pipeline-test.properties index e11c5f099..cadc78ebb 100644 --- a/eva-accession-clustering/src/test/resources/clustering-pipeline-test.properties +++ b/eva-accession-clustering/src/test/resources/clustering-pipeline-test.properties @@ -6,6 +6,7 @@ parameters.assemblyAccession=GCA_000000001.1 parameters.remappedFrom=ASM1 parameters.rsReportPath=src/test/resources/output-files/rsReport.txt parameters.chunkSize=2 +parameters.rsAccFile=src/test/resources/input-files/rsAcc.txt eva.count-stats.url=http://localhost:8080 eva.count-stats.username=username @@ -23,9 +24,6 @@ accessioning.monotonic.rs.nextBlockInterval=1000000000 spring.data.mongodb.database=test-db -spring.data.mongodb.host=|eva.mongo.host.test| -spring.data.mongodb.password= -spring.data.mongodb.port=27017 mongodb.read-preference=primary # See https://github.com/spring-projects/spring-boot/wiki/Spring-Boot-2.1-Release-Notes#bean-overriding diff --git a/eva-accession-clustering/src/test/resources/clustering-qc-test.properties b/eva-accession-clustering/src/test/resources/clustering-qc-test.properties index 35fad5052..4b654722b 100644 --- a/eva-accession-clustering/src/test/resources/clustering-qc-test.properties +++ b/eva-accession-clustering/src/test/resources/clustering-qc-test.properties @@ -23,9 +23,6 @@ accessioning.monotonic.rs.nextBlockInterval=1000000000 spring.data.mongodb.database=test-db -spring.data.mongodb.host=|eva.mongo.host.test| -spring.data.mongodb.password= -spring.data.mongodb.port=27017 mongodb.read-preference=primary # See https://github.com/spring-projects/spring-boot/wiki/Spring-Boot-2.1-Release-Notes#bean-overriding diff --git a/eva-accession-clustering/src/test/resources/clustering-writer-test.properties b/eva-accession-clustering/src/test/resources/clustering-writer-test.properties index 30cb01327..652d34dd3 100644 --- a/eva-accession-clustering/src/test/resources/clustering-writer-test.properties +++ b/eva-accession-clustering/src/test/resources/clustering-writer-test.properties @@ -24,9 +24,6 @@ accessioning.monotonic.rs.nextBlockInterval=1000000000 spring.data.mongodb.database=test-db -spring.data.mongodb.host=|eva.mongo.host.test| -spring.data.mongodb.password= -spring.data.mongodb.port=27017 mongodb.read-preference=primary # See https://github.com/spring-projects/spring-boot/wiki/Spring-Boot-2.1-Release-Notes#bean-overriding diff --git a/eva-accession-clustering/src/test/resources/duplicate-rs-acc-qc-test.properties b/eva-accession-clustering/src/test/resources/duplicate-rs-acc-qc-test.properties index 7fd3d655b..6c0cf5110 100644 --- a/eva-accession-clustering/src/test/resources/duplicate-rs-acc-qc-test.properties +++ b/eva-accession-clustering/src/test/resources/duplicate-rs-acc-qc-test.properties @@ -25,9 +25,6 @@ accessioning.monotonic.rs.nextBlockInterval=1000000000 spring.data.mongodb.database=test-db -spring.data.mongodb.host=|eva.mongo.host.test| -spring.data.mongodb.password= -spring.data.mongodb.port=27017 mongodb.read-preference=primary # See https://github.com/spring-projects/spring-boot/wiki/Spring-Boot-2.1-Release-Notes#bean-overriding diff --git a/eva-accession-clustering/src/test/resources/merge-split-test.properties b/eva-accession-clustering/src/test/resources/merge-split-test.properties index 8740229b5..3aea699b4 100644 --- a/eva-accession-clustering/src/test/resources/merge-split-test.properties +++ b/eva-accession-clustering/src/test/resources/merge-split-test.properties @@ -22,9 +22,6 @@ accessioning.monotonic.rs.nextBlockInterval=1000000000 spring.data.mongodb.database=test-db -spring.data.mongodb.host=|eva.mongo.host.test| -spring.data.mongodb.password= -spring.data.mongodb.port=27017 mongodb.read-preference=primary # See https://github.com/spring-projects/spring-boot/wiki/Spring-Boot-2.1-Release-Notes#bean-overriding diff --git a/eva-accession-clustering/src/test/resources/rs-accession-recovery.properties b/eva-accession-clustering/src/test/resources/rs-accession-recovery.properties index a66292426..b5418f3f3 100644 --- a/eva-accession-clustering/src/test/resources/rs-accession-recovery.properties +++ b/eva-accession-clustering/src/test/resources/rs-accession-recovery.properties @@ -4,9 +4,10 @@ spring.datasource.driver-class-name=org.hsqldb.jdbcDriver spring.datasource.url=jdbc:hsqldb:mem:db;sql.syntax_pgs=true;DB_CLOSE_DELAY=-1 spring.datasource.username=SA spring.datasource.password= -spring.datasource.schema=test-data/contiguous_id_blocks_schema.sql -spring.datasource.data=test-data/rs_accession_recovery_test_data.sql -spring.jpa.hibernate.ddl-auto=update +spring.sql.init.schema-locations=classpath:test-data/contiguous_id_blocks_schema.sql +spring.sql.init.data-locations=classpath:test-data/rs_accession_recovery_test_data.sql +spring.sql.init.mode=always +spring.jpa.hibernate.ddl-auto=none parameters.vcf=src/test/resources/input-files/vcf/aggregated_accessioned.vcf.gz parameters.projectAccession=projectId_1 @@ -31,9 +32,6 @@ accessioning.monotonic.rs.blockStartValue=3000000000 accessioning.monotonic.rs.nextBlockInterval=1000000000 spring.data.mongodb.database=test-db -spring.data.mongodb.host=|eva.mongo.host.test| -spring.data.mongodb.password= -spring.data.mongodb.port=27017 mongodb.read-preference=primary # See https://github.com/spring-projects/spring-boot/wiki/Spring-Boot-2.1-Release-Notes#bean-overriding diff --git a/eva-accession-core/pom.xml b/eva-accession-core/pom.xml index beddfbe6d..9335ac7ea 100644 --- a/eva-accession-core/pom.xml +++ b/eva-accession-core/pom.xml @@ -5,7 +5,7 @@ uk.ac.ebi.eva eva-accession - 0.6.57-SNAPSHOT + ${revision} eva-accession-core @@ -49,8 +49,18 @@ accession-commons-monotonic-generator-jpa - com.lordofthejars - nosqlunit-mongodb + org.springframework.boot + spring-boot-testcontainers + test + + + org.testcontainers + mongodb + test + + + org.testcontainers + junit-jupiter test @@ -91,8 +101,4 @@ - - localhost - - diff --git a/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/batch/io/ClusteredVariantDeprecationWriter.java b/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/batch/io/ClusteredVariantDeprecationWriter.java index fe0abc075..232e40a37 100644 --- a/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/batch/io/ClusteredVariantDeprecationWriter.java +++ b/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/batch/io/ClusteredVariantDeprecationWriter.java @@ -20,11 +20,12 @@ import org.apache.commons.lang3.tuple.ImmutablePair; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.batch.item.Chunk; import org.springframework.batch.item.ItemWriter; import org.springframework.data.mongodb.core.MongoTemplate; import uk.ac.ebi.ampt2d.commons.accession.core.models.EventType; import uk.ac.ebi.ampt2d.commons.accession.persistence.mongodb.document.EventDocument; - +import uk.ac.ebi.eva.accession.core.EVAObjectModelUtils; import uk.ac.ebi.eva.accession.core.model.IClusteredVariant; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpClusteredVariantEntity; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpClusteredVariantOperationEntity; @@ -33,7 +34,6 @@ import uk.ac.ebi.eva.accession.core.model.eva.ClusteredVariantOperationEntity; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; import uk.ac.ebi.eva.accession.core.service.nonhuman.SubmittedVariantAccessioningService; -import uk.ac.ebi.eva.accession.core.EVAObjectModelUtils; import java.util.Arrays; import java.util.Collections; @@ -67,7 +67,7 @@ public ClusteredVariantDeprecationWriter(String assemblyAccession, MongoTemplate SubmittedVariantAccessioningService submittedVariantAccessioningService, Long accessioningMonotonicInitRs, String deprecationIdSuffix, String deprecationReason) { - ReadPreference readPreference = mongoTemplate.getMongoDbFactory().getDb().getReadPreference(); + ReadPreference readPreference = mongoTemplate.getMongoDatabaseFactory().getMongoDatabase().getReadPreference(); if (!readPreference.equals(ReadPreference.primary())) { throw new IllegalStateException("Read preference setting should be primary to deprecate variants!"); } @@ -84,49 +84,51 @@ public int getNumDeprecatedEntities() { } @Override - public void write(List cvesToDeprecate) { - List cvesToDeprecateInCVE = cvesToDeprecate.stream().filter( + public void write(Chunk cvesToDeprecate) { + List cvesToDeprecateInCVE = cvesToDeprecate.getItems().stream().filter( cve -> (cve.getAccession() >= accessioningMonotonicInitRs)).collect(Collectors.toList()); - List cvesToDeprecateInDbsnpCVE = cvesToDeprecate.stream().filter( + List cvesToDeprecateInDbsnpCVE = cvesToDeprecate.getItems().stream().filter( cve -> (cve.getAccession() < accessioningMonotonicInitRs)).collect(Collectors.toList()); deprecateVariants(cvesToDeprecateInCVE, ClusteredVariantEntity.class); deprecateVariants(cvesToDeprecateInDbsnpCVE, DbsnpClusteredVariantEntity.class); } private void deprecateVariants(List cvesToDeprecate, - Class cveCollectionToUse) { + Class cveCollectionToUse) { if (cvesToDeprecate.size() > 0) { Set> rsHashesAndIDsToRemove = cvesToDeprecate.stream().map( - cve -> new ImmutablePair<>(cve.getHashedMessage(), cve.getAccession())).collect(Collectors.toSet()); + cve -> new ImmutablePair<>(cve.getHashedMessage(), cve.getAccession())) + .collect(Collectors.toSet()); Set> rsHashesAndIDsAssociatedWithExistingSS = this.submittedVariantAccessioningService.getByClusteredVariantAccessionIn( - rsHashesAndIDsToRemove.stream().map(c -> c.right).collect(Collectors.toList())).stream() + rsHashesAndIDsToRemove.stream().map(c -> c.right) + .collect(Collectors.toList())).stream() .map(result -> new SubmittedVariantEntity(result.getAccession(), result.getHash(), result.getData(), result.getVersion())) .filter(result -> result.getReferenceSequenceAccession().equals(this.assemblyAccession) && rsHashesAndIDsToRemove.contains( - new ImmutablePair<>(EVAObjectModelUtils.getClusteredVariantHash(result), - result.getClusteredVariantAccession()))) + new ImmutablePair<>(EVAObjectModelUtils.getClusteredVariantHash(result), + result.getClusteredVariantAccession()))) .map(result -> new ImmutablePair<>(EVAObjectModelUtils.getClusteredVariantHash(result), result.getClusteredVariantAccession())) .collect(Collectors.toSet()); if (rsHashesAndIDsAssociatedWithExistingSS.size() > 0) { logger.warn("The following RS ID/hash combinations are still associated with existing submitted variants. " + - "Hence they will not be deprecated. The combinations are: " + + "Hence they will not be deprecated. The combinations are: " + rsHashesAndIDsAssociatedWithExistingSS.stream().map(Object::toString).collect( Collectors.joining(","))); } rsHashesAndIDsToRemove.removeAll(rsHashesAndIDsAssociatedWithExistingSS); cvesToDeprecate = cvesToDeprecate.stream() - .filter(cve -> - rsHashesAndIDsToRemove.contains(new ImmutablePair<>( - cve.getHashedMessage(), cve.getAccession()))) - .collect(Collectors.toList()); + .filter(cve -> + rsHashesAndIDsToRemove.contains(new ImmutablePair<>( + cve.getHashedMessage(), cve.getAccession()))) + .collect(Collectors.toList()); Set rsHashesToRemove = cvesToDeprecate.stream() - .map(ClusteredVariantEntity::getId) - .collect(Collectors.toSet()); + .map(ClusteredVariantEntity::getId) + .collect(Collectors.toSet()); Class> cvoeCollectionToUse = cveCollectionToUse.equals(ClusteredVariantEntity.class) ? ClusteredVariantOperationEntity.class : DbsnpClusteredVariantOperationEntity.class; @@ -138,20 +140,20 @@ private void deprecateVariants(List cvesToDepr } private void writeDeprecationOperation(List cvesToDeprecate, - Class> cvoeCollectionToUse) { + Class> cvoeCollectionToUse) { List cvoesToWrite = cvesToDeprecate.stream().map(cve -> { ClusteredVariantOperationEntity cvoe = new ClusteredVariantOperationEntity(); cvoe.fill(EventType.DEPRECATED, cve.getAccession(), null, - this.deprecationReason, - Collections.singletonList(new ClusteredVariantInactiveEntity(cve))); + this.deprecationReason, + Collections.singletonList(new ClusteredVariantInactiveEntity(cve))); cvoe.setId(String.join("_", Arrays.asList("RS_DEPRECATED", this.deprecationIdSuffix, cve.getId()))); return cvoe; }).collect(Collectors.toList()); Set operationIdsToWrite = cvoesToWrite.stream().map(EventDocument::getId).collect(Collectors.toSet()); Set alreadyExistingIds = this.mongoTemplate.find(query(where("_id").in(operationIdsToWrite)), - cvoeCollectionToUse).stream().map(EventDocument::getId) - .collect(Collectors.toSet()); + cvoeCollectionToUse).stream().map(EventDocument::getId) + .collect(Collectors.toSet()); operationIdsToWrite.removeAll(alreadyExistingIds); cvoesToWrite = cvoesToWrite.stream().filter(svoe -> operationIdsToWrite.contains(svoe.getId())).collect( Collectors.toList()); diff --git a/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/batch/io/DbsnpClusteredVariantOperationWriter.java b/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/batch/io/DbsnpClusteredVariantOperationWriter.java index 992c3f4c3..6c5097725 100644 --- a/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/batch/io/DbsnpClusteredVariantOperationWriter.java +++ b/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/batch/io/DbsnpClusteredVariantOperationWriter.java @@ -20,15 +20,13 @@ import com.mongodb.MongoBulkWriteException; import com.mongodb.bulk.BulkWriteResult; +import org.springframework.batch.item.Chunk; import org.springframework.batch.item.ItemWriter; import org.springframework.dao.DuplicateKeyException; import org.springframework.data.mongodb.core.BulkOperations; import org.springframework.data.mongodb.core.MongoTemplate; - -import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpClusteredVariantOperationEntity; import uk.ac.ebi.eva.accession.core.batch.listeners.ImportCounts; - -import java.util.List; +import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpClusteredVariantOperationEntity; public class DbsnpClusteredVariantOperationWriter implements ItemWriter { @@ -42,12 +40,12 @@ public DbsnpClusteredVariantOperationWriter(MongoTemplate mongoTemplate, ImportC } @Override - public void write(List importedClusteredVariantsOperations) + public void write(Chunk importedClusteredVariantsOperations) throws Exception { try { BulkOperations bulkOperations = mongoTemplate.bulkOps(BulkOperations.BulkMode.UNORDERED, - DbsnpClusteredVariantOperationEntity.class); - bulkOperations.insert(importedClusteredVariantsOperations); + DbsnpClusteredVariantOperationEntity.class); + bulkOperations.insert(importedClusteredVariantsOperations.getItems()); bulkOperations.execute(); importCounts.addOperationsWritten(importedClusteredVariantsOperations.size()); } catch (DuplicateKeyException exception) { diff --git a/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/batch/io/DbsnpClusteredVariantWriter.java b/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/batch/io/DbsnpClusteredVariantWriter.java index 65f94de05..0231a7b29 100644 --- a/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/batch/io/DbsnpClusteredVariantWriter.java +++ b/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/batch/io/DbsnpClusteredVariantWriter.java @@ -17,14 +17,13 @@ import com.mongodb.MongoBulkWriteException; import com.mongodb.bulk.BulkWriteResult; +import org.springframework.batch.item.Chunk; import org.springframework.batch.item.ItemWriter; -import org.springframework.dao.DuplicateKeyException; +import org.springframework.data.mongodb.BulkOperationException; import org.springframework.data.mongodb.core.BulkOperations; import org.springframework.data.mongodb.core.MongoTemplate; -import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpClusteredVariantEntity; import uk.ac.ebi.eva.accession.core.batch.listeners.ImportCounts; - -import java.util.List; +import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpClusteredVariantEntity; public class DbsnpClusteredVariantWriter implements ItemWriter { @@ -38,14 +37,14 @@ public DbsnpClusteredVariantWriter(MongoTemplate mongoTemplate, ImportCounts imp } @Override - public void write(List importedClusteredVariants) { + public void write(Chunk importedClusteredVariants) { try { BulkOperations bulkOperations = mongoTemplate.bulkOps(BulkOperations.BulkMode.UNORDERED, - DbsnpClusteredVariantEntity.class); - bulkOperations.insert(importedClusteredVariants); + DbsnpClusteredVariantEntity.class); + bulkOperations.insert(importedClusteredVariants.getItems()); bulkOperations.execute(); importCounts.addClusteredVariantsWritten(importedClusteredVariants.size()); - } catch (DuplicateKeyException exception) { + } catch (BulkOperationException exception) { BulkWriteResult bulkWriteResult = ((MongoBulkWriteException) exception.getCause()).getWriteResult(); importCounts.addClusteredVariantsWritten(bulkWriteResult.getInsertedCount()); throw exception; diff --git a/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/batch/io/SubmittedVariantDeprecationWriter.java b/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/batch/io/SubmittedVariantDeprecationWriter.java index d7ef84dd2..10e22bf82 100644 --- a/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/batch/io/SubmittedVariantDeprecationWriter.java +++ b/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/batch/io/SubmittedVariantDeprecationWriter.java @@ -18,12 +18,13 @@ import com.mongodb.ReadPreference; import org.apache.commons.lang3.tuple.ImmutablePair; +import org.springframework.batch.item.Chunk; import org.springframework.batch.item.ItemWriter; import org.springframework.data.mongodb.core.MongoTemplate; import uk.ac.ebi.ampt2d.commons.accession.core.models.EventType; import uk.ac.ebi.ampt2d.commons.accession.persistence.mongodb.document.AccessionedDocument; import uk.ac.ebi.ampt2d.commons.accession.persistence.mongodb.document.EventDocument; - +import uk.ac.ebi.eva.accession.core.EVAObjectModelUtils; import uk.ac.ebi.eva.accession.core.model.ISubmittedVariant; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpSubmittedVariantEntity; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpSubmittedVariantOperationEntity; @@ -33,9 +34,12 @@ import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantOperationEntity; import uk.ac.ebi.eva.accession.core.service.nonhuman.ClusteredVariantAccessioningService; import uk.ac.ebi.eva.accession.core.service.nonhuman.SubmittedVariantAccessioningService; -import uk.ac.ebi.eva.accession.core.EVAObjectModelUtils; -import java.util.*; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.Set; import java.util.stream.Collectors; import static org.springframework.data.mongodb.core.query.Criteria.where; @@ -64,7 +68,7 @@ public SubmittedVariantDeprecationWriter(String assemblyAccession, MongoTemplate ClusteredVariantAccessioningService clusteredVariantAccessioningService, Long accessioningMonotonicInitSs, Long accessioningMonotonicInitRs, String deprecationIdSuffix, String deprecationReason) { - ReadPreference readPreference = mongoTemplate.getMongoDbFactory().getDb().getReadPreference(); + ReadPreference readPreference = mongoTemplate.getMongoDatabaseFactory().getMongoDatabase().getReadPreference(); if (!readPreference.equals(ReadPreference.primary())) { throw new IllegalStateException("Read preference setting should be primary to deprecate variants!"); } @@ -76,25 +80,27 @@ public SubmittedVariantDeprecationWriter(String assemblyAccession, MongoTemplate this.deprecationReason = deprecationReason; this.clusteredVariantDeprecationWriter = new ClusteredVariantDeprecationWriter(this.assemblyAccession, - this.mongoTemplate, - submittedVariantAccessioningService, - accessioningMonotonicInitRs, - this.deprecationIdSuffix, - this.deprecationReason); + this.mongoTemplate, + submittedVariantAccessioningService, + accessioningMonotonicInitRs, + this.deprecationIdSuffix, + this.deprecationReason); } @Override - public void write(List svesToDeprecate) { - List svesToDeprecateInSVE = svesToDeprecate.stream().filter( - sve -> (sve.getAccession() >= accessioningMonotonicInitSs)).collect(Collectors.toList()); - List svesToDeprecateInDbsnpSVE = svesToDeprecate.stream().filter( - sve -> (sve.getAccession() < accessioningMonotonicInitSs)).collect(Collectors.toList()); + public void write(Chunk svesToDeprecate) { + List svesToDeprecateInSVE = svesToDeprecate.getItems().stream().filter( + sve -> (sve.getAccession() >= accessioningMonotonicInitSs)) + .collect(Collectors.toList()); + List svesToDeprecateInDbsnpSVE = svesToDeprecate.getItems().stream().filter( + sve -> (sve.getAccession() < accessioningMonotonicInitSs)) + .collect(Collectors.toList()); deprecateVariants(svesToDeprecateInSVE, SubmittedVariantEntity.class); deprecateVariants(svesToDeprecateInDbsnpSVE, DbsnpSubmittedVariantEntity.class); } private void deprecateVariants(List svesToDeprecate, - Class sveCollectionToUse) { + Class sveCollectionToUse) { if (svesToDeprecate.size() > 0) { List ssHashesToRemove = svesToDeprecate.stream().map(AccessionedDocument::getId).collect( Collectors.toList()); @@ -111,7 +117,7 @@ private void deprecateVariants(List svesToDepr associatedRSHashesAndIDs.contains( new ImmutablePair<>(result.getHash(), result.getAccession()))) .map(result -> new ClusteredVariantEntity(result.getAccession(), result.getHash(), - result.getData())) + result.getData())) .collect(Collectors.toList()); Class> svoeCollectionToUse = sveCollectionToUse.equals(SubmittedVariantEntity.class) ? @@ -120,25 +126,25 @@ private void deprecateVariants(List svesToDepr List removedEntities = this.mongoTemplate.findAllAndRemove( query(where("_id").in(ssHashesToRemove)), sveCollectionToUse); this.numDeprecatedSubmittedEntities += removedEntities.size(); - this.clusteredVariantDeprecationWriter.write(cvesToDeprecate); + this.clusteredVariantDeprecationWriter.write(new Chunk<>(cvesToDeprecate)); } } private void writeDeprecationOperation(List svesToDeprecate, - Class> svoeCollectionToUse) { + Class> svoeCollectionToUse) { List svoesToWrite = svesToDeprecate.stream().map(sve -> { SubmittedVariantOperationEntity svoe = new SubmittedVariantOperationEntity(); svoe.fill(EventType.DEPRECATED, sve.getAccession(), null, - this.deprecationReason, - Collections.singletonList(new SubmittedVariantInactiveEntity(sve))); + this.deprecationReason, + Collections.singletonList(new SubmittedVariantInactiveEntity(sve))); svoe.setId(String.join("_", Arrays.asList("SS_DEPRECATED", this.deprecationIdSuffix, sve.getId()))); return svoe; }).collect(Collectors.toList()); Set operationIds = svoesToWrite.stream().map(EventDocument::getId).collect(Collectors.toSet()); Set alreadyExistingIds = this.mongoTemplate.find(query(where("_id").in(operationIds)), - svoeCollectionToUse).stream().map(EventDocument::getId) - .collect(Collectors.toSet()); + svoeCollectionToUse).stream().map(EventDocument::getId) + .collect(Collectors.toSet()); operationIds.removeAll(alreadyExistingIds); svoesToWrite = svoesToWrite.stream().filter(svoe -> operationIds.contains(svoe.getId())).collect( Collectors.toList()); diff --git a/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/batch/listeners/GenericProgressListener.java b/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/batch/listeners/GenericProgressListener.java index e14abd58a..5d6c1fed2 100644 --- a/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/batch/listeners/GenericProgressListener.java +++ b/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/batch/listeners/GenericProgressListener.java @@ -22,8 +22,7 @@ import org.springframework.batch.core.StepExecution; import org.springframework.batch.core.listener.StepListenerSupport; import org.springframework.batch.core.scope.context.ChunkContext; - -import java.util.List; +import org.springframework.batch.item.Chunk; public class GenericProgressListener extends StepListenerSupport { @@ -64,14 +63,14 @@ public void afterRead(I itemRead) { } @Override - public void beforeWrite(List items) { + public void beforeWrite(Chunk items) { logger.debug("About to write chunk"); } @Override - public void afterWrite(List items) { + public void afterWrite(Chunk items) { if (items.size() > 0) { - O lastItem = items.get(items.size() - 1); + O lastItem = items.getItems().get(items.size() - 1); logger.debug("Written chunk of {} items. Last item was {}: {}", items.size(), lastItem.toString()); } else { logger.debug("Written chunk of 0 items."); diff --git a/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/batch/policies/IllegalStartSkipPolicy.java b/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/batch/policies/IllegalStartSkipPolicy.java index a4f8c5a92..e40490c15 100644 --- a/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/batch/policies/IllegalStartSkipPolicy.java +++ b/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/batch/policies/IllegalStartSkipPolicy.java @@ -23,7 +23,7 @@ public class IllegalStartSkipPolicy implements SkipPolicy { @Override - public boolean shouldSkip(Throwable exception, int skipCount) throws SkipLimitExceededException { + public boolean shouldSkip(Throwable exception, long skipCount) throws SkipLimitExceededException { return exception instanceof PositionOutsideOfContigException; } } diff --git a/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/configuration/MongoClientCreator.java b/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/configuration/MongoClientCreator.java index 221e1148c..48fc310ed 100644 --- a/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/configuration/MongoClientCreator.java +++ b/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/configuration/MongoClientCreator.java @@ -1,41 +1,44 @@ package uk.ac.ebi.eva.accession.core.configuration; -import com.mongodb.*; +import com.mongodb.ConnectionString; +import com.mongodb.MongoClientSettings; +import com.mongodb.ReadConcern; +import com.mongodb.ReadPreference; +import com.mongodb.WriteConcern; +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoClients; import org.springframework.beans.factory.ObjectProvider; -import org.springframework.boot.autoconfigure.mongo.MongoClientFactory; import org.springframework.boot.autoconfigure.mongo.MongoProperties; -import org.springframework.core.env.Environment; import uk.ac.ebi.eva.commons.mongodb.utils.MongoUtils; -import java.io.UnsupportedEncodingException; import java.util.Objects; public class MongoClientCreator { - public static MongoClient getMongoClient(MongoProperties properties, ObjectProvider options, - Environment environment, String readPreference) - throws UnsupportedEncodingException { - MongoClientOptions mongoClientOptions = options.getIfAvailable(); + public static MongoClient getMongoClient(MongoProperties properties, ObjectProvider settings, + String readPreference) { + MongoClientSettings mongoClientSettings = settings.getIfAvailable(); // Only set the URI if it isn't already set if (Objects.isNull(properties.getUri())) { // Weirdly, MongoClient instantiation works without authentication mechanism // in the eva-accession project but does not work in the eva-pipeline project // So we explicitly pass it as null here - properties.setUri(MongoUtils.constructMongoClientURI(properties.getHost(), properties.getPort(), + properties.setUri(MongoUtils.constructMongoConnectionString(properties.getHost(), properties.getPort(), properties.getDatabase(), properties.getUsername(), (Objects.nonNull(properties.getPassword()) ? String.valueOf(properties.getPassword()) : ""), - properties.getAuthenticationDatabase(), null, readPreference).getURI()); + properties.getAuthenticationDatabase(), null, readPreference).getConnectionString()); } // If we don't do this Spring gets confused since both the URI (which already has the host) // and the host parameters are present properties.setHost(null); - MongoClientOptions.Builder mongoClientOptionsBuilder; - if (mongoClientOptions != null) { - mongoClientOptionsBuilder = new MongoClientOptions.Builder(mongoClientOptions); + MongoClientSettings.Builder mongoClientSettingsBuilder; + if (mongoClientSettings != null) { + mongoClientSettingsBuilder = MongoClientSettings.builder(mongoClientSettings); } else { - mongoClientOptionsBuilder = new MongoClientOptions.Builder(); + mongoClientSettingsBuilder = MongoClientSettings.builder(); } - mongoClientOptions = mongoClientOptionsBuilder.readPreference(ReadPreference.valueOf(readPreference)) - .writeConcern(WriteConcern.MAJORITY).readConcern(ReadConcern.MAJORITY).build(); - return new MongoClientFactory(properties, environment).createMongoClient(mongoClientOptions); + mongoClientSettings = mongoClientSettingsBuilder.readPreference(ReadPreference.valueOf(readPreference)) + .writeConcern(WriteConcern.MAJORITY).readConcern(ReadConcern.MAJORITY) + .applyConnectionString(new ConnectionString(properties.getUri())).build(); + return MongoClients.create(mongoClientSettings); } } diff --git a/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/configuration/human/HumanMongoConfiguration.java b/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/configuration/human/HumanMongoConfiguration.java index 847d0ad24..5876c486c 100644 --- a/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/configuration/human/HumanMongoConfiguration.java +++ b/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/configuration/human/HumanMongoConfiguration.java @@ -15,8 +15,8 @@ */ package uk.ac.ebi.eva.accession.core.configuration.human; -import com.mongodb.MongoClient; -import com.mongodb.MongoClientOptions; +import com.mongodb.MongoClientSettings; +import com.mongodb.client.MongoClient; import org.springframework.beans.factory.ObjectProvider; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Value; @@ -25,12 +25,11 @@ import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import org.springframework.core.env.Environment; import org.springframework.data.domain.Sort; -import org.springframework.data.mongodb.MongoDbFactory; +import org.springframework.data.mongodb.MongoDatabaseFactory; import org.springframework.data.mongodb.config.EnableMongoAuditing; import org.springframework.data.mongodb.core.MongoTemplate; -import org.springframework.data.mongodb.core.SimpleMongoDbFactory; +import org.springframework.data.mongodb.core.SimpleMongoClientDatabaseFactory; import org.springframework.data.mongodb.core.WriteResultChecking; import org.springframework.data.mongodb.core.convert.DefaultDbRefResolver; import org.springframework.data.mongodb.core.convert.DefaultMongoTypeMapper; @@ -63,30 +62,28 @@ public MongoProperties mongoProperties() { @Bean("humanMongoClient") public MongoClient mongoClient(@Qualifier("humanMongoProperties") MongoProperties properties, - ObjectProvider options, Environment environment) + ObjectProvider settings) throws UnknownHostException, UnsupportedEncodingException { - return MongoClientCreator.getMongoClient(properties, options, environment, readPreference); + return MongoClientCreator.getMongoClient(properties, settings, readPreference); } @Bean("humanFactory") - public MongoDbFactory mongoDbFactory(@Qualifier("humanMongoProperties") MongoProperties properties, - ObjectProvider options, - Environment environment) + public MongoDatabaseFactory mongoDbFactory(@Qualifier("humanMongoProperties") MongoProperties properties, + ObjectProvider settings) throws UnknownHostException, UnsupportedEncodingException { - return new SimpleMongoDbFactory(mongoClient(properties, options, environment), properties.getDatabase()); + return new SimpleMongoClientDatabaseFactory(mongoClient(properties, settings), properties.getDatabase()); } @Bean("humanMappingConverter") public MappingMongoConverter mappingMongoConverter(MongoProperties properties, - ObjectProvider options, - Environment environment) + ObjectProvider settings) throws UnknownHostException, UnsupportedEncodingException { - return new MappingMongoConverter(new DefaultDbRefResolver(mongoDbFactory(properties, options, environment)), - new MongoMappingContext()); + return new MappingMongoConverter(new DefaultDbRefResolver(mongoDbFactory(properties, settings)), + new MongoMappingContext()); } @Bean(name = "humanMongoTemplate") - public MongoTemplate humanMongoTemplate(@Qualifier("humanFactory") MongoDbFactory mongoDbFactory, + public MongoTemplate humanMongoTemplate(@Qualifier("humanFactory") MongoDatabaseFactory mongoDbFactory, @Qualifier("humanMappingConverter") MappingMongoConverter converter) { converter.setTypeMapper(new DefaultMongoTypeMapper(null)); MongoTemplate mongoTemplate = new MongoTemplate(mongoDbFactory, converter); diff --git a/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/configuration/nonhuman/MongoConfiguration.java b/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/configuration/nonhuman/MongoConfiguration.java index a4c6864f7..4c44b31f4 100644 --- a/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/configuration/nonhuman/MongoConfiguration.java +++ b/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/configuration/nonhuman/MongoConfiguration.java @@ -15,11 +15,10 @@ */ package uk.ac.ebi.eva.accession.core.configuration.nonhuman; -import com.mongodb.MongoClient; -import com.mongodb.MongoClientOptions; - +import com.mongodb.MongoClientSettings; import com.mongodb.ReadPreference; import com.mongodb.WriteConcern; +import com.mongodb.client.MongoClient; import org.springframework.beans.factory.ObjectProvider; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Value; @@ -29,21 +28,22 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Primary; -import org.springframework.core.env.Environment; -import org.springframework.data.mongodb.MongoDbFactory; +import org.springframework.data.mongodb.MongoDatabaseFactory; import org.springframework.data.mongodb.config.EnableMongoAuditing; import org.springframework.data.mongodb.core.MongoTemplate; -import org.springframework.data.mongodb.core.SimpleMongoDbFactory; +import org.springframework.data.mongodb.core.SimpleMongoClientDatabaseFactory; import org.springframework.data.mongodb.core.WriteResultChecking; import org.springframework.data.mongodb.core.convert.DefaultDbRefResolver; import org.springframework.data.mongodb.core.convert.DefaultMongoTypeMapper; import org.springframework.data.mongodb.core.convert.MappingMongoConverter; +import org.springframework.data.mongodb.core.convert.MongoCustomConversions; import org.springframework.data.mongodb.core.mapping.MongoMappingContext; import org.springframework.data.mongodb.repository.config.EnableMongoRepositories; import uk.ac.ebi.eva.accession.core.configuration.MongoClientCreator; import java.io.UnsupportedEncodingException; import java.net.UnknownHostException; +import java.util.Collections; @Configuration @EnableMongoRepositories(basePackages = {"uk.ac.ebi.eva.accession.core.repository"}) @@ -66,33 +66,50 @@ public MongoProperties mongoProperties() { @Bean @Primary - public MongoClient mongoClient(MongoProperties properties, ObjectProvider options, - Environment environment) throws UnknownHostException, UnsupportedEncodingException { - return MongoClientCreator.getMongoClient(properties, options, environment, readPreference); + public MongoClient mongoClient(MongoProperties properties, ObjectProvider settings) + throws UnknownHostException, UnsupportedEncodingException { + return MongoClientCreator.getMongoClient(properties, settings, readPreference); } @Bean("primaryFactory") @Primary - public MongoDbFactory mongoDbFactory(MongoProperties properties, - ObjectProvider options, - Environment environment) + public MongoDatabaseFactory mongoDbFactory(MongoProperties properties, ObjectProvider settings) throws UnknownHostException, UnsupportedEncodingException { - return new SimpleMongoDbFactory(mongoClient(properties, options, environment), properties.getDatabase()); + return new SimpleMongoClientDatabaseFactory(mongoClient(properties, settings), properties.getDatabase()); + } + + @Bean + @Primary + public MongoCustomConversions mongoCustomConversions() { + return new MongoCustomConversions(Collections.emptyList()); + } + + @Bean + @Primary + public MongoMappingContext mongoMappingContext(MongoCustomConversions conversions) { + MongoMappingContext context = new MongoMappingContext(); + context.setSimpleTypeHolder(conversions.getSimpleTypeHolder()); + context.setAutoIndexCreation(false); + return context; } @Bean @Primary public MappingMongoConverter mappingMongoConverter(MongoProperties properties, - ObjectProvider options, - Environment environment) + ObjectProvider settings, + MongoMappingContext mongoMappingContext, + MongoCustomConversions conversions) throws UnknownHostException, UnsupportedEncodingException { - return new MappingMongoConverter(new DefaultDbRefResolver(mongoDbFactory(properties, options, environment)), - new MongoMappingContext()); + MappingMongoConverter converter = new MappingMongoConverter(new DefaultDbRefResolver(mongoDbFactory(properties, settings)), + mongoMappingContext); + converter.setCustomConversions(conversions); + converter.afterPropertiesSet(); + return converter; } @Primary @Bean - public MongoTemplate mongoTemplate(@Qualifier("primaryFactory") MongoDbFactory mongoDbFactory, + public MongoTemplate mongoTemplate(@Qualifier("primaryFactory") MongoDatabaseFactory mongoDbFactory, MappingMongoConverter converter) { converter.setTypeMapper(new DefaultMongoTypeMapper(null)); MongoTemplate mongoTemplate = new MongoTemplate(mongoDbFactory, converter); diff --git a/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/configuration/nonhuman/SubmittedVariantAccessioningConfiguration.java b/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/configuration/nonhuman/SubmittedVariantAccessioningConfiguration.java index e0e1c7840..5e9e17e98 100644 --- a/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/configuration/nonhuman/SubmittedVariantAccessioningConfiguration.java +++ b/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/configuration/nonhuman/SubmittedVariantAccessioningConfiguration.java @@ -24,31 +24,31 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Import; +import org.springframework.context.annotation.Lazy; import uk.ac.ebi.ampt2d.commons.accession.autoconfigure.EnableSpringDataContiguousIdService; import uk.ac.ebi.ampt2d.commons.accession.core.AccessionSaveMode; import uk.ac.ebi.ampt2d.commons.accession.generators.monotonic.MonotonicAccessionGenerator; import uk.ac.ebi.ampt2d.commons.accession.hashing.SHA1HashingFunction; import uk.ac.ebi.ampt2d.commons.accession.persistence.jpa.monotonic.service.ContiguousIdBlockService; - import uk.ac.ebi.eva.accession.core.configuration.ApplicationProperties; import uk.ac.ebi.eva.accession.core.configuration.ApplicationPropertiesConfiguration; import uk.ac.ebi.eva.accession.core.configuration.ContigAliasConfiguration; import uk.ac.ebi.eva.accession.core.contigalias.ContigAliasService; -import uk.ac.ebi.eva.accession.core.model.ISubmittedVariant; -import uk.ac.ebi.eva.accession.core.service.nonhuman.SubmittedVariantAccessioningService; import uk.ac.ebi.eva.accession.core.generators.DbsnpMonotonicAccessionGenerator; -import uk.ac.ebi.eva.accession.core.service.nonhuman.dbsnp.DbsnpSubmittedVariantAccessioningDatabaseService; -import uk.ac.ebi.eva.accession.core.repository.nonhuman.dbsnp.DbsnpSubmittedVariantAccessioningRepository; +import uk.ac.ebi.eva.accession.core.model.ISubmittedVariant; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpSubmittedVariantInactiveEntity; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpSubmittedVariantOperationEntity; -import uk.ac.ebi.eva.accession.core.repository.nonhuman.dbsnp.DbsnpSubmittedVariantOperationRepository; -import uk.ac.ebi.eva.accession.core.service.nonhuman.eva.SubmittedVariantAccessioningDatabaseService; -import uk.ac.ebi.eva.accession.core.repository.nonhuman.eva.SubmittedVariantAccessioningRepository; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantInactiveEntity; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantOperationEntity; +import uk.ac.ebi.eva.accession.core.repository.nonhuman.dbsnp.DbsnpSubmittedVariantAccessioningRepository; +import uk.ac.ebi.eva.accession.core.repository.nonhuman.dbsnp.DbsnpSubmittedVariantOperationRepository; +import uk.ac.ebi.eva.accession.core.repository.nonhuman.eva.SubmittedVariantAccessioningRepository; import uk.ac.ebi.eva.accession.core.repository.nonhuman.eva.SubmittedVariantOperationRepository; +import uk.ac.ebi.eva.accession.core.service.nonhuman.SubmittedVariantAccessioningService; +import uk.ac.ebi.eva.accession.core.service.nonhuman.dbsnp.DbsnpSubmittedVariantAccessioningDatabaseService; import uk.ac.ebi.eva.accession.core.service.nonhuman.dbsnp.DbsnpSubmittedVariantInactiveService; import uk.ac.ebi.eva.accession.core.service.nonhuman.dbsnp.DbsnpSubmittedVariantMonotonicAccessioningService; +import uk.ac.ebi.eva.accession.core.service.nonhuman.eva.SubmittedVariantAccessioningDatabaseService; import uk.ac.ebi.eva.accession.core.service.nonhuman.eva.SubmittedVariantInactiveService; import uk.ac.ebi.eva.accession.core.service.nonhuman.eva.SubmittedVariantMonotonicAccessioningService; import uk.ac.ebi.eva.accession.core.summary.SubmittedVariantSummaryFunction; @@ -72,9 +72,11 @@ public class SubmittedVariantAccessioningConfiguration { @Autowired private DbsnpSubmittedVariantOperationRepository dbsnpOperationRepository; + @Lazy @Autowired private SubmittedVariantInactiveService inactiveService; + @Lazy @Autowired private DbsnpSubmittedVariantInactiveService dbsnpInactiveService; diff --git a/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/model/ClusteredVariant.java b/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/model/ClusteredVariant.java index 533baf811..30e7c8091 100644 --- a/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/model/ClusteredVariant.java +++ b/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/model/ClusteredVariant.java @@ -17,11 +17,11 @@ */ package uk.ac.ebi.eva.accession.core.model; +import uk.ac.ebi.eva.commons.core.models.VariantType; + import java.time.LocalDateTime; import java.util.Objects; -import uk.ac.ebi.eva.commons.core.models.VariantType; - public class ClusteredVariant implements IClusteredVariant { private String assemblyAccession; diff --git a/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/model/dbsnp/DbsnpSubmittedVariantInactiveEntity.java b/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/model/dbsnp/DbsnpSubmittedVariantInactiveEntity.java index 83744bd2a..0c45081ea 100644 --- a/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/model/dbsnp/DbsnpSubmittedVariantInactiveEntity.java +++ b/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/model/dbsnp/DbsnpSubmittedVariantInactiveEntity.java @@ -17,7 +17,6 @@ */ package uk.ac.ebi.eva.accession.core.model.dbsnp; -import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpSubmittedVariantEntity; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantInactiveEntity; public class DbsnpSubmittedVariantInactiveEntity extends SubmittedVariantInactiveEntity { diff --git a/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/runner/CommandLineRunnerUtils.java b/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/runner/CommandLineRunnerUtils.java index fc6226c7b..589886bf4 100644 --- a/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/runner/CommandLineRunnerUtils.java +++ b/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/runner/CommandLineRunnerUtils.java @@ -16,6 +16,7 @@ import uk.ac.ebi.eva.commons.batch.job.JobStatusManager; import java.util.Comparator; +import java.util.HashMap; import java.util.List; import java.util.Map; @@ -26,11 +27,11 @@ public class CommandLineRunnerUtils { private static final String RUN_ID_PARAMETER_NAME = "run.id"; public static JobParameters addRunIDToJobParameters(String jobName, JobExplorer jobExplorer, - JobParameters jobParameters) { + JobParameters jobParameters) { JobExecution lastJobExecution = getLastJobExecution(jobName, jobExplorer, jobParameters); if (lastJobExecution != null) { Long runIdParameterFromLastExecution = lastJobExecution.getJobParameters() - .getLong(RUN_ID_PARAMETER_NAME); + .getLong(RUN_ID_PARAMETER_NAME); if (runIdParameterFromLastExecution != 0 && lastJobExecution.getStatus() == BatchStatus.FAILED) { // Spring Batch 4 uses all job parameters (including run.id) to detect previous instances of a job - see https://github.com/spring-projects/spring-boot/blob/86fb39d5c5f474fe3544159270d4c4e2d01d43ef/spring-boot-project/spring-boot-autoconfigure/src/main/java/org/springframework/boot/autoconfigure/batch/JobLauncherCommandLineRunner.java#L222 // as opposed to Spring 3 which uses only job name - see https://github.com/spring-projects/spring-boot/blob/541890f0e003a6e346f2234102c97105ab1292ee/spring-boot-autoconfigure/src/main/java/org/springframework/boot/autoconfigure/batch/JobLauncherCommandLineRunner.java#L131 @@ -43,16 +44,16 @@ public static JobParameters addRunIDToJobParameters(String jobName, JobExplorer return jobParameters; } - public static JobExecution getLastJobExecution (String jobName, JobExplorer jobExplorer, - JobParameters previousJobParameters) { - int previousJobInstanceCount = getPreviousJobInstanceCount(jobName, jobExplorer); + public static JobExecution getLastJobExecution(String jobName, JobExplorer jobExplorer, + JobParameters previousJobParameters) { + long previousJobInstanceCount = getPreviousJobInstanceCount(jobName, jobExplorer); - List jobInstanceList = jobExplorer.getJobInstances(jobName, 0, previousJobInstanceCount); + List jobInstanceList = jobExplorer.getJobInstances(jobName, 0, (int) previousJobInstanceCount); List matchingJobExecutions; - for (JobInstance jobInstance: jobInstanceList) { + for (JobInstance jobInstance : jobInstanceList) { matchingJobExecutions = jobExplorer.getJobExecutions(jobInstance); - for (JobExecution jobExecution: matchingJobExecutions) { + for (JobExecution jobExecution : matchingJobExecutions) { if (areParametersEquivalentExceptRunId(jobExecution.getJobParameters(), previousJobParameters)) { return matchingJobExecutions.stream().max(Comparator.comparingLong(Entity::getId)).get(); } @@ -61,18 +62,17 @@ public static JobExecution getLastJobExecution (String jobName, JobExplorer jobE return null; } - private static int getPreviousJobInstanceCount(String jobName, JobExplorer jobExplorer) { + private static long getPreviousJobInstanceCount(String jobName, JobExplorer jobExplorer) { try { return jobExplorer.getJobInstanceCount(jobName); - } - catch (NoSuchJobException ex) { + } catch (NoSuchJobException ex) { return 0; } } - private static boolean areParametersEquivalentExceptRunId (JobParameters parameters1, JobParameters parameters2) { - Map firstJobParameterMap = parameters1.getParameters(); - Map secondJobParameterMap = parameters2.getParameters(); + private static boolean areParametersEquivalentExceptRunId(JobParameters parameters1, JobParameters parameters2) { + Map> firstJobParameterMap = new HashMap<>(parameters1.getParameters()); + Map> secondJobParameterMap = new HashMap<>(parameters2.getParameters()); if (parameters1.getParameters().containsKey(RUN_ID_PARAMETER_NAME)) { firstJobParameterMap.remove(RUN_ID_PARAMETER_NAME); } @@ -87,8 +87,7 @@ public static void markPreviousJobAsFailed(String jobName, JobRepository jobRepo logger.info("Force restartPreviousExecution of job '" + jobName + "' with parameters: " + jobParameters); try { JobStatusManager.markLastJobAsFailed(jobRepository, jobName, jobParameters); - } - catch (NoPreviousJobExecutionException ex) { + } catch (NoPreviousJobExecutionException ex) { //Be forgiving if a previous job does not exist: just print a warning message logger.warn(ex.getMessage()); } diff --git a/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/service/nonhuman/dbsnp/DbsnpSubmittedVariantInactiveService.java b/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/service/nonhuman/dbsnp/DbsnpSubmittedVariantInactiveService.java index 3bc3f6d80..2a3265e5b 100644 --- a/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/service/nonhuman/dbsnp/DbsnpSubmittedVariantInactiveService.java +++ b/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/service/nonhuman/dbsnp/DbsnpSubmittedVariantInactiveService.java @@ -17,9 +17,8 @@ */ package uk.ac.ebi.eva.accession.core.service.nonhuman.dbsnp; -import uk.ac.ebi.ampt2d.commons.accession.persistence.repositories.IHistoryRepository; import uk.ac.ebi.ampt2d.commons.accession.persistence.mongodb.service.BasicMongoDbInactiveAccessionService; - +import uk.ac.ebi.ampt2d.commons.accession.persistence.repositories.IHistoryRepository; import uk.ac.ebi.eva.accession.core.model.ISubmittedVariant; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpSubmittedVariantEntity; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpSubmittedVariantInactiveEntity; diff --git a/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/service/nonhuman/eva/SubmittedVariantInactiveService.java b/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/service/nonhuman/eva/SubmittedVariantInactiveService.java index f04cbade5..9505c27be 100644 --- a/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/service/nonhuman/eva/SubmittedVariantInactiveService.java +++ b/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/service/nonhuman/eva/SubmittedVariantInactiveService.java @@ -17,9 +17,8 @@ */ package uk.ac.ebi.eva.accession.core.service.nonhuman.eva; -import uk.ac.ebi.ampt2d.commons.accession.persistence.repositories.IHistoryRepository; import uk.ac.ebi.ampt2d.commons.accession.persistence.mongodb.service.BasicMongoDbInactiveAccessionService; - +import uk.ac.ebi.ampt2d.commons.accession.persistence.repositories.IHistoryRepository; import uk.ac.ebi.eva.accession.core.model.ISubmittedVariant; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantInactiveEntity; diff --git a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/EVAObjectModelUtilsTest.java b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/EVAObjectModelUtilsTest.java index 4f609a14c..da09a904c 100644 --- a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/EVAObjectModelUtilsTest.java +++ b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/EVAObjectModelUtilsTest.java @@ -15,15 +15,18 @@ */ package uk.ac.ebi.eva.accession.core; -import org.junit.Test; +import org.junit.jupiter.api.Test; import uk.ac.ebi.eva.accession.core.model.ClusteredVariant; import uk.ac.ebi.eva.accession.core.model.SubmittedVariant; import uk.ac.ebi.eva.accession.core.model.eva.ClusteredVariantEntity; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; import uk.ac.ebi.eva.commons.core.models.VariantType; -import static org.junit.Assert.assertEquals; -import static uk.ac.ebi.eva.accession.core.EVAObjectModelUtils.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static uk.ac.ebi.eva.accession.core.EVAObjectModelUtils.getClusteredVariantHash; +import static uk.ac.ebi.eva.accession.core.EVAObjectModelUtils.toClusteredVariant; +import static uk.ac.ebi.eva.accession.core.EVAObjectModelUtils.toClusteredVariantEntity; +import static uk.ac.ebi.eva.accession.core.EVAObjectModelUtils.toSubmittedVariantEntity; public class EVAObjectModelUtilsTest { diff --git a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/batch/io/AccessionedVcfFactoryTest.java b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/batch/io/AccessionedVcfFactoryTest.java index e479d4061..4422c9caf 100644 --- a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/batch/io/AccessionedVcfFactoryTest.java +++ b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/batch/io/AccessionedVcfFactoryTest.java @@ -1,15 +1,20 @@ package uk.ac.ebi.eva.accession.core.batch.io; -import org.junit.BeforeClass; -import org.junit.Test; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; import uk.ac.ebi.eva.commons.core.models.factories.VariantVcfFactory; import uk.ac.ebi.eva.commons.core.models.pipeline.Variant; -import java.util.*; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; public class AccessionedVcfFactoryTest { @@ -20,7 +25,7 @@ public class AccessionedVcfFactoryTest { private static VariantVcfFactory factory; - @BeforeClass + @BeforeAll public static void setupClass() { factory = new AccessionedVcfFactory(); } diff --git a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/batch/io/ClusteredVariantDeprecationWriterTest.java b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/batch/io/ClusteredVariantDeprecationWriterTest.java index 224bd39dd..0e864cedb 100644 --- a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/batch/io/ClusteredVariantDeprecationWriterTest.java +++ b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/batch/io/ClusteredVariantDeprecationWriterTest.java @@ -16,23 +16,18 @@ package uk.ac.ebi.eva.accession.core.batch.io; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; -import com.mongodb.MongoClient; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.batch.item.Chunk; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; -import org.springframework.context.ApplicationContext; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.ampt2d.commons.accession.hashing.SHA1HashingFunction; - import uk.ac.ebi.eva.accession.core.configuration.nonhuman.ClusteredVariantAccessioningConfiguration; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.MongoConfiguration; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.SubmittedVariantAccessioningConfiguration; @@ -47,24 +42,21 @@ import uk.ac.ebi.eva.accession.core.service.nonhuman.SubmittedVariantAccessioningService; import uk.ac.ebi.eva.accession.core.summary.ClusteredVariantSummaryFunction; import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; -import uk.ac.ebi.eva.accession.core.test.rule.FixSpringMongoDbRule; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.commons.core.models.pipeline.Variant; import java.util.Arrays; import java.util.function.Function; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; -@RunWith(SpringRunner.class) +@ExtendWith(SpringExtension.class) @TestPropertySource("classpath:ss-deprecation-test.properties") @EnableAutoConfiguration @ContextConfiguration(classes = {MongoConfiguration.class, MongoTestConfiguration.class, SubmittedVariantAccessioningConfiguration.class, ClusteredVariantAccessioningConfiguration.class}) -public class ClusteredVariantDeprecationWriterTest { - - private static final String TEST_DB = "sve-deprecation-test"; - +public class ClusteredVariantDeprecationWriterTest extends MongoTestContainerHelper { private static final String ASSEMBLY = "GCA_000000001.1"; private static final int TAXONOMY = 60711; @@ -72,9 +64,6 @@ public class ClusteredVariantDeprecationWriterTest { @Autowired private Long accessioningMonotonicInitRs; - @Autowired - private MongoClient mongoClient; - @Autowired private MongoTemplate mongoTemplate; @@ -84,26 +73,14 @@ public class ClusteredVariantDeprecationWriterTest { private SubmittedVariantEntity ss1, ss2, ss3, ss4; private ClusteredVariantEntity rs1, rs2; - //Required by nosql-unit - @Autowired - private ApplicationContext applicationContext; - - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName(TEST_DB).build()); - - private void cleanup() { - mongoClient.dropDatabase(TEST_DB); - } - - @Before + @BeforeEach public void setUp() { - cleanup(); + mongoTemplate.getDb().drop(); } - @After + @AfterEach public void tearDown() { - cleanup(); + mongoTemplate.getDb().drop(); } @Test @@ -125,18 +102,18 @@ public void testDeprecateRS() { ClusteredVariantDeprecationWriter cveDeprecationWriter = new ClusteredVariantDeprecationWriter(ASSEMBLY, this.mongoTemplate, - this.submittedVariantAccessioningService, - this.accessioningMonotonicInitRs, - "TEST", "Deprecation test"); + this.submittedVariantAccessioningService, + this.accessioningMonotonicInitRs, + "TEST", "Deprecation test"); this.mongoTemplate.remove(ss1, this.mongoTemplate.getCollectionName(DbsnpSubmittedVariantEntity.class)); this.mongoTemplate.remove(ss2, this.mongoTemplate.getCollectionName(DbsnpSubmittedVariantEntity.class)); this.mongoTemplate.remove(ss3, this.mongoTemplate.getCollectionName(SubmittedVariantEntity.class)); - cveDeprecationWriter.write(Arrays.asList(rs1, rs2)); + cveDeprecationWriter.write(Chunk.of(rs1, rs2)); assertPostDeprecationDatabaseState(); // Ensure that the second run of deprecation does not do any harm - cveDeprecationWriter.write(Arrays.asList(rs1, rs2)); + cveDeprecationWriter.write(Chunk.of(rs1, rs2)); assertPostDeprecationDatabaseState(); } @@ -149,7 +126,7 @@ private void assertPostDeprecationDatabaseState() { assertEquals(0, this.mongoTemplate.findAll(ClusteredVariantOperationEntity.class).size()); DbsnpClusteredVariantOperationEntity rs1DeprecationOp = this.mongoTemplate.findById("RS_DEPRECATED_TEST_" + rs1.getHashedMessage(), - DbsnpClusteredVariantOperationEntity.class); + DbsnpClusteredVariantOperationEntity.class); assertNotNull(rs1DeprecationOp); assertEquals(rs1, rs1DeprecationOp.getInactiveObjects().get(0).toClusteredVariantEntity()); } @@ -158,20 +135,20 @@ private SubmittedVariantEntity createSS(Long ssAccession, Long rsAccession, Long String alternate) { return new SubmittedVariantEntity(ssAccession, "hash" + ssAccession, ASSEMBLY, TAXONOMY, - "PRJ1", "chr1", start, reference, alternate, rsAccession, false, false, false, - false, 1); + "PRJ1", "chr1", start, reference, alternate, rsAccession, false, false, false, + false, 1); } private ClusteredVariantEntity createRS(SubmittedVariantEntity sve) { - Function hashingFunction = new ClusteredVariantSummaryFunction().andThen( + Function hashingFunction = new ClusteredVariantSummaryFunction().andThen( new SHA1HashingFunction()); ClusteredVariant cv = new ClusteredVariant(sve.getReferenceSequenceAccession(), sve.getTaxonomyAccession(), - sve.getContig(), - sve.getStart(), - new Variant(sve.getContig(), sve.getStart(), sve.getStart(), - sve.getReferenceAllele(), - sve.getAlternateAllele()).getType(), - true, null); + sve.getContig(), + sve.getStart(), + new Variant(sve.getContig(), sve.getStart(), sve.getStart(), + sve.getReferenceAllele(), + sve.getAlternateAllele()).getType(), + true, null); String hash = hashingFunction.apply(cv); return new ClusteredVariantEntity(sve.getClusteredVariantAccession(), hash, cv); } diff --git a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/batch/io/DbsnpClusteredVariantWriterTest.java b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/batch/io/DbsnpClusteredVariantWriterTest.java index 86f4c5be1..35f2313a1 100644 --- a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/batch/io/DbsnpClusteredVariantWriterTest.java +++ b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/batch/io/DbsnpClusteredVariantWriterTest.java @@ -15,44 +15,45 @@ */ package uk.ac.ebi.eva.accession.core.batch.io; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.batch.item.Chunk; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.dao.DuplicateKeyException; +import org.springframework.data.mongodb.BulkOperationException; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.data.mongodb.core.query.Query; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.ampt2d.commons.accession.hashing.SHA1HashingFunction; - +import uk.ac.ebi.eva.accession.core.batch.listeners.ImportCounts; import uk.ac.ebi.eva.accession.core.model.ClusteredVariant; import uk.ac.ebi.eva.accession.core.model.IClusteredVariant; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpClusteredVariantEntity; import uk.ac.ebi.eva.accession.core.summary.ClusteredVariantSummaryFunction; -import uk.ac.ebi.eva.accession.core.batch.listeners.ImportCounts; import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import java.time.LocalDateTime; import java.time.Month; import java.util.Arrays; -import java.util.Collections; import java.util.List; import java.util.function.Function; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertThrows; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; import static uk.ac.ebi.eva.commons.core.models.VariantType.INDEL; import static uk.ac.ebi.eva.commons.core.models.VariantType.SNV; -@RunWith(SpringRunner.class) +@ExtendWith(SpringExtension.class) @TestPropertySource("classpath:test-variants-writer.properties") @ContextConfiguration(classes = {MongoTestConfiguration.class}) -public class DbsnpClusteredVariantWriterTest { +public class DbsnpClusteredVariantWriterTest extends MongoTestContainerHelper { private static final int TAXONOMY_1 = 3880; @@ -93,8 +94,9 @@ public class DbsnpClusteredVariantWriterTest { private ClusteredVariant clusteredVariant3; - @Before + @BeforeEach public void setUp() { + mongoTemplate.getDb().drop(); importCounts = new ImportCounts(); dbsnpClusteredVariantWriter = new DbsnpClusteredVariantWriter(mongoTemplate, importCounts); hashingFunction = new ClusteredVariantSummaryFunction().andThen(new SHA1HashingFunction()); @@ -103,8 +105,8 @@ public void setUp() { // variants and entity objects clusteredVariant1 = new ClusteredVariant(ASSEMBLY, TAXONOMY_1, CONTIG, START, SNV, true, CREATED_DATE); ClusteredVariant duplicateClusteredVariant1 = new ClusteredVariant(ASSEMBLY, TAXONOMY_1, CONTIG, START, SNV, - false, null); - clusteredVariant2 = new ClusteredVariant(ASSEMBLY, TAXONOMY_1, CONTIG, START+1, SNV, true, null); + false, null); + clusteredVariant2 = new ClusteredVariant(ASSEMBLY, TAXONOMY_1, CONTIG, START + 1, SNV, true, null); clusteredVariant3 = new ClusteredVariant(ASSEMBLY, TAXONOMY_1, CONTIG, START, INDEL, true, null); variantEntity1 = buildClusteredVariantEntity(ACCESSION_1, clusteredVariant1); duplicateVariantEntity1 = buildClusteredVariantEntity(ACCESSION_1, duplicateClusteredVariant1); @@ -112,37 +114,42 @@ public void setUp() { variantEntity3 = buildClusteredVariantEntity(ACCESSION_3, clusteredVariant3); } + @AfterEach + public void tearDown() { + mongoTemplate.getDb().drop(); + } + private DbsnpClusteredVariantEntity buildClusteredVariantEntity(long accession, ClusteredVariant clusteredVariant) { return new DbsnpClusteredVariantEntity(accession, hashingFunction.apply(clusteredVariant), clusteredVariant); } @Test public void saveSingleAccession() { - dbsnpClusteredVariantWriter.write(Collections.singletonList(variantEntity1)); + dbsnpClusteredVariantWriter.write(Chunk.of(variantEntity1)); assertJustOneVariantHasBeenStored(); } private void assertJustOneVariantHasBeenStored() { List storedVariants = mongoTemplate.find(new Query(), - DbsnpClusteredVariantEntity.class); + DbsnpClusteredVariantEntity.class); assertEquals(1, storedVariants.size()); assertTrue(storedVariants.contains(variantEntity1)); assertEquals(ACCESSION_1, storedVariants.get(0).getAccession()); assertEquals(clusteredVariant1, storedVariants.get(0).getModel()); assertEquals(1, importCounts.getClusteredVariantsWritten()); - assertEquals(CREATED_DATE, storedVariants.get(0).getModel().getCreatedDate()); } @Test public void exceptionThrownOnDuplicateIdenticalVariant() { - assertThrows(DuplicateKeyException.class, () -> - dbsnpClusteredVariantWriter.write(Arrays.asList(variantEntity1, variantEntity1))); + BulkOperationException exception = assertThrows(BulkOperationException.class, () -> + dbsnpClusteredVariantWriter.write(Chunk.of(variantEntity1, variantEntity1))); + assertTrue(exception.getMessage().contains("duplicate key error")); } @Test public void duplicateIdenticalVariantIsStoredJustOnce() { try { - dbsnpClusteredVariantWriter.write(Arrays.asList(variantEntity1, variantEntity1)); + dbsnpClusteredVariantWriter.write(Chunk.of(variantEntity1, variantEntity1)); fail(); } catch (Exception e) { // it's correct and expected that an exception is thrown here @@ -154,7 +161,7 @@ public void duplicateIdenticalVariantIsStoredJustOnce() { @Test public void duplicateNotIdenticalVariantIsStoredJustOnce() { try { - dbsnpClusteredVariantWriter.write(Arrays.asList(variantEntity1, duplicateVariantEntity1)); + dbsnpClusteredVariantWriter.write(Chunk.of(variantEntity1, duplicateVariantEntity1)); fail(); } catch (Exception e) { // it's correct and expected that an exception is thrown here @@ -164,13 +171,13 @@ public void duplicateNotIdenticalVariantIsStoredJustOnce() { @Test public void saveDifferentVariants() { - dbsnpClusteredVariantWriter.write(Arrays.asList(variantEntity1, variantEntity2, variantEntity3)); + dbsnpClusteredVariantWriter.write(Chunk.of(variantEntity1, variantEntity2, variantEntity3)); assertAllUniqueVariantsHaveBeenStored(); } private void assertAllUniqueVariantsHaveBeenStored() { List storedVariants = mongoTemplate.find(new Query(), - DbsnpClusteredVariantEntity.class); + DbsnpClusteredVariantEntity.class); assertEquals(3, storedVariants.size()); assertFalse(storedVariants.contains(duplicateVariantEntity1)); assertEquals(ACCESSION_1, storedVariants.get(0).getAccession()); @@ -185,10 +192,10 @@ private void assertAllUniqueVariantsHaveBeenStored() { @Test public void allNonDuplicatedRecordsWillBeWritten() { List batch = Arrays.asList(variantEntity1, variantEntity2, duplicateVariantEntity1, - variantEntity3); + variantEntity3); try { - dbsnpClusteredVariantWriter.write(batch); + dbsnpClusteredVariantWriter.write(new Chunk<>(batch)); fail(); } catch (Exception e) { // it's correct and expected that an exception is thrown here diff --git a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/batch/io/FastaSequenceReaderTest.java b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/batch/io/FastaSequenceReaderTest.java index 5fdfb50ee..fc748d7d3 100644 --- a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/batch/io/FastaSequenceReaderTest.java +++ b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/batch/io/FastaSequenceReaderTest.java @@ -16,13 +16,11 @@ package uk.ac.ebi.eva.accession.core.batch.io; import org.apache.commons.lang3.tuple.ImmutableTriple; -import org.junit.After; -import org.junit.Before; -import org.junit.Ignore; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; -import org.junit.rules.TemporaryFolder; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; +import uk.ac.ebi.eva.accession.core.utils.PipelineTemporaryFolderUtil; import java.io.File; import java.io.IOException; @@ -31,7 +29,8 @@ import java.nio.file.Path; import java.nio.file.Paths; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; public class FastaSequenceReaderTest { @@ -39,62 +38,54 @@ public class FastaSequenceReaderTest { private FastaSequenceReader reader; - @Rule - public final ExpectedException thrown = ExpectedException.none(); + public PipelineTemporaryFolderUtil temporaryFolderUtil = new PipelineTemporaryFolderUtil(); - @Rule - public TemporaryFolder temporaryFolder = new TemporaryFolder(); - - @Before + @BeforeEach public void setUp() throws Exception { reader = new FastaSequenceReader(Paths.get( FastaSequenceReaderTest.class.getResource("/input-files/fasta/Gallus_gallus-5.0.test.fa").toURI())); } - @After + @AfterEach public void tearDown() throws Exception { reader.close(); } @Test - public void getFirstNucleotideOfContig() throws Exception { + public void getFirstNucleotideOfContig() { assertEquals("T", reader.getSequence("22", 1, 1)); } @Test - public void getLastNucleotideOfContig() throws Exception { + public void getLastNucleotideOfContig() { assertEquals("G", reader.getSequence("22", 4729743, 4729743)); } @Test - public void getSequence() throws Exception { + public void getSequence() { // this sequence is split between three lines in the FASTA file assertEquals("GTTTCAAGTGGTTGTGACCCCCGCTGCACAGTCAGTTGGGTTAGGGTTAGGGTTAGGGTCAGTCACAGTCAGTTGTCAGACTGGTGTTTA", - reader.getSequence("22", 59986, 60075)); + reader.getSequence("22", 59986, 60075)); } @Test - public void endMustBeGreaterOrEqualsThanStart() throws Exception { - thrown.expect(IllegalArgumentException.class); - reader.getSequence("22", 1000, 999); + public void endMustBeGreaterOrEqualsThanStart() { + assertThrows(IllegalArgumentException.class, () -> reader.getSequence("22", 1000, 999)); } @Test - public void onlyPositiveCoordinatesAreAllowed() throws Exception { - thrown.expect(IllegalArgumentException.class); - reader.getSequence("22", -1, 5); + public void onlyPositiveCoordinatesAreAllowed() { + assertThrows(IllegalArgumentException.class, () -> reader.getSequence("22", -1, 5)); } @Test - public void coordinatesGreaterThanEndOfChromosomeAreNotAllowed() throws Exception { - thrown.expect(IllegalArgumentException.class); - reader.getSequence("22", 4729740, 4729750); + public void coordinatesGreaterThanEndOfChromosomeAreNotAllowed() { + assertThrows(IllegalArgumentException.class, () -> reader.getSequence("22", 4729740, 4729750)); } @Test - public void notExistentChromosome() throws Exception { - thrown.expect(IllegalArgumentException.class); - reader.getSequence("23", 1, 1); + public void notExistentChromosome() { + assertThrows(IllegalArgumentException.class, () -> reader.getSequence("23", 1, 1)); } @Test @@ -103,7 +94,7 @@ public void fastaWithNoDictionary() throws Exception { FastaSequenceReaderTest.class.getResource("/input-files/fasta/fastaWithNoDictionary.fa").toURI())); // this sequence is split between three lines in the FASTA file assertEquals("CAGCCGCAGTCCGGACAGCGCATGCGCCAGCCGCGAGACCGCACAGCGCATGCGCCAGCGCGAGTGACAGCG", - fastaSequenceReader.getSequence("22", 174, 245)); + fastaSequenceReader.getSequence("22", 174, 245)); } @Test @@ -113,11 +104,11 @@ public void fastaWithNoIndex() throws Exception { // this sequence is split between three lines in the FASTA file assertEquals("CAGCCGCAGTCCGGACAGCGCATGCGCCAGCCGCGAGACCGCACAGCGCATGCGCCAGCGCGAGTGACAGCG", - fastaSequenceReader.getSequence("22", 174, 245)); + fastaSequenceReader.getSequence("22", 174, 245)); } private FastaSequenceReader getFastaSequenceReader(String fastaFilename) throws IOException, URISyntaxException { - File temporaryFolderRoot = temporaryFolder.getRoot(); + File temporaryFolderRoot = temporaryFolderUtil.getRoot(); Path fasta = Files.copy( Paths.get(FastaSequenceReaderTest.class.getResource("/input-files/fasta/" + fastaFilename).toURI()), temporaryFolderRoot.toPath().resolve(fastaFilename)); @@ -132,7 +123,7 @@ private FastaSequenceReader getFastaSequenceReader(String fastaFilename) throws * forbids compressed fastas. You can comment the requirement by hand and run this test to see if it still applies. */ @Test - @Ignore + @Disabled public void htsDoesNotSupportCompressedFastas() throws URISyntaxException, IOException { String fastaFilename = "compressed.fa.gz"; FastaSequenceReader fastaSequenceReader = getFastaSequenceReader(fastaFilename); @@ -142,18 +133,17 @@ public void htsDoesNotSupportCompressedFastas() throws URISyntaxException, IOExc /** * For the rationale of this test, look at {@link #htsDoesNotSupportCompressedFastas()} and - * {@link FastaSequenceReader#checkFastaIsUncompressed(java.nio.file.Path)} + * {@link FastaSequenceReader#checkFastaIsUncompressed(java.nio.file.Path)} */ @Test public void shouldThrowOnCompressedFasta() throws URISyntaxException, IOException { String fastaFilename = "compressed.fa.gz"; - File temporaryFolderRoot = temporaryFolder.getRoot(); + File temporaryFolderRoot = temporaryFolderUtil.getRoot(); Path fasta = Files.copy( Paths.get(FastaSequenceReaderTest.class.getResource("/input-files/fasta/" + fastaFilename).toURI()), temporaryFolderRoot.toPath().resolve(fastaFilename)); - thrown.expect(IllegalArgumentException.class); - new FastaSequenceReader(fasta); + assertThrows(IllegalArgumentException.class, () -> new FastaSequenceReader(fasta)); } @Test diff --git a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/batch/io/SubmittedVariantDeprecationWriterTest.java b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/batch/io/SubmittedVariantDeprecationWriterTest.java index dcbd7f824..8179d4a33 100644 --- a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/batch/io/SubmittedVariantDeprecationWriterTest.java +++ b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/batch/io/SubmittedVariantDeprecationWriterTest.java @@ -16,23 +16,18 @@ package uk.ac.ebi.eva.accession.core.batch.io; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; -import com.mongodb.MongoClient; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.batch.item.Chunk; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; -import org.springframework.context.ApplicationContext; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.ampt2d.commons.accession.hashing.SHA1HashingFunction; - import uk.ac.ebi.eva.accession.core.configuration.nonhuman.ClusteredVariantAccessioningConfiguration; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.MongoConfiguration; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.SubmittedVariantAccessioningConfiguration; @@ -50,21 +45,21 @@ import uk.ac.ebi.eva.accession.core.service.nonhuman.SubmittedVariantAccessioningService; import uk.ac.ebi.eva.accession.core.summary.ClusteredVariantSummaryFunction; import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; -import uk.ac.ebi.eva.accession.core.test.rule.FixSpringMongoDbRule; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.commons.core.models.pipeline.Variant; import java.util.Arrays; import java.util.function.Function; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; -@RunWith(SpringRunner.class) +@ExtendWith(SpringExtension.class) @TestPropertySource("classpath:ss-deprecation-test.properties") @EnableAutoConfiguration @ContextConfiguration(classes = {MongoConfiguration.class, MongoTestConfiguration.class, SubmittedVariantAccessioningConfiguration.class, ClusteredVariantAccessioningConfiguration.class}) -public class SubmittedVariantDeprecationWriterTest { +public class SubmittedVariantDeprecationWriterTest extends MongoTestContainerHelper { private static final String TEST_DB = "sve-deprecation-test"; @@ -80,9 +75,6 @@ public class SubmittedVariantDeprecationWriterTest { @Autowired private Long accessioningMonotonicInitRs; - @Autowired - private MongoClient mongoClient; - @Autowired private MongoTemplate mongoTemplate; @@ -95,26 +87,14 @@ public class SubmittedVariantDeprecationWriterTest { private SubmittedVariantEntity ss1, ss2, ss3, ss4; private ClusteredVariantEntity rs1, rs1AnotherHash, rs2; - //Required by nosql-unit - @Autowired - private ApplicationContext applicationContext; - - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName(TEST_DB).build()); - - private void cleanup() { - mongoClient.dropDatabase(TEST_DB); - } - - @Before + @BeforeEach public void setUp() { - cleanup(); + mongoTemplate.getDb().drop(); } - @After + @AfterEach public void tearDown() { - cleanup(); + mongoTemplate.getDb().drop(); } @Test @@ -136,16 +116,16 @@ public void testDeprecateSS() { SubmittedVariantDeprecationWriter sveDeprecationWriter = new SubmittedVariantDeprecationWriter(ASSEMBLY, this.mongoTemplate, - this.submittedVariantAccessioningService, - this.clusteredVariantAccessioningService, - this.accessioningMonotonicInitSs, - this.accessioningMonotonicInitRs, - "TEST", REASON); - sveDeprecationWriter.write(Arrays.asList(ss1, ss2, ss3)); + this.submittedVariantAccessioningService, + this.clusteredVariantAccessioningService, + this.accessioningMonotonicInitSs, + this.accessioningMonotonicInitRs, + "TEST", REASON); + sveDeprecationWriter.write(Chunk.of(ss1, ss2, ss3)); assertPostDeprecationDatabaseState(); // Ensure that the second run of deprecation does not do any harm - sveDeprecationWriter.write(Arrays.asList(ss1, ss2, ss3)); + sveDeprecationWriter.write(Chunk.of(ss1, ss2, ss3)); assertPostDeprecationDatabaseState(); } @@ -180,7 +160,7 @@ private void assertPostDeprecationDatabaseState() { assertEquals(0, this.mongoTemplate.findAll(ClusteredVariantOperationEntity.class).size()); DbsnpClusteredVariantOperationEntity rs1DeprecationOp = this.mongoTemplate.findById("RS_DEPRECATED_TEST_" + rs1.getHashedMessage(), - DbsnpClusteredVariantOperationEntity.class); + DbsnpClusteredVariantOperationEntity.class); assertNotNull(rs1DeprecationOp); assertEquals(rs1, rs1DeprecationOp.getInactiveObjects().get(0).toClusteredVariantEntity()); } @@ -189,20 +169,20 @@ private SubmittedVariantEntity createSS(Long ssAccession, Long rsAccession, Long String alternate) { return new SubmittedVariantEntity(ssAccession, "hash" + ssAccession, ASSEMBLY, TAXONOMY, - "PRJ1", "chr1", start, reference, alternate, rsAccession, false, false, false, - false, 1); + "PRJ1", "chr1", start, reference, alternate, rsAccession, false, false, false, + false, 1); } private ClusteredVariantEntity createRS(SubmittedVariantEntity sve) { - Function hashingFunction = new ClusteredVariantSummaryFunction().andThen( + Function hashingFunction = new ClusteredVariantSummaryFunction().andThen( new SHA1HashingFunction()); ClusteredVariant cv = new ClusteredVariant(sve.getReferenceSequenceAccession(), sve.getTaxonomyAccession(), - sve.getContig(), - sve.getStart(), - new Variant(sve.getContig(), sve.getStart(), sve.getStart(), - sve.getReferenceAllele(), - sve.getAlternateAllele()).getType(), - true, null); + sve.getContig(), + sve.getStart(), + new Variant(sve.getContig(), sve.getStart(), sve.getStart(), + sve.getReferenceAllele(), + sve.getAlternateAllele()).getType(), + true, null); String hash = hashingFunction.apply(cv); return new ClusteredVariantEntity(sve.getClusteredVariantAccession(), hash, cv); } @@ -235,11 +215,11 @@ public void testDeprecateSSWithSpecificRSHash() { this.accessioningMonotonicInitSs, this.accessioningMonotonicInitRs, "TEST", REASON); - sveDeprecationWriter.write(Arrays.asList(ss1, ss2, ss3)); + sveDeprecationWriter.write(Chunk.of(ss1, ss2, ss3)); assertPostDeprecationDatabaseStateForSpecificRSHash(); // Ensure that the second run of deprecation does not do any harm - sveDeprecationWriter.write(Arrays.asList(ss1, ss2, ss3)); + sveDeprecationWriter.write(Chunk.of(ss1, ss2, ss3)); assertPostDeprecationDatabaseStateForSpecificRSHash(); } diff --git a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/batch/policies/IllegalStartSkipPolicyTest.java b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/batch/policies/IllegalStartSkipPolicyTest.java index 34e4b7dca..186989b86 100644 --- a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/batch/policies/IllegalStartSkipPolicyTest.java +++ b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/batch/policies/IllegalStartSkipPolicyTest.java @@ -15,19 +15,19 @@ */ package uk.ac.ebi.eva.accession.core.batch.policies; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import uk.ac.ebi.eva.accession.core.exceptions.PositionOutsideOfContigException; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; public class IllegalStartSkipPolicyTest { private IllegalStartSkipPolicy illegalStartSkipPolicy; - @Before + @BeforeEach public void setUp() throws Exception { illegalStartSkipPolicy = new IllegalStartSkipPolicy(); } diff --git a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/contig/ContigMappingTest.java b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/contig/ContigMappingTest.java index 45ab7832f..4061ea353 100644 --- a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/contig/ContigMappingTest.java +++ b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/contig/ContigMappingTest.java @@ -15,13 +15,13 @@ */ package uk.ac.ebi.eva.accession.core.contig; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; public class ContigMappingTest { @@ -71,7 +71,7 @@ public class ContigMappingTest { private static final int MISSING_UCSC_ROWS = 1; - @Before + @BeforeEach public void setUp() throws Exception { String fileString = ContigMappingTest.class.getResource( "/input-files/assembly-report/GCA_000001635.8_Mus_musculus-grcm38.p6_assembly_report.txt").toString(); diff --git a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/model/eva/ClusteredVariantEntityIntegrationTest.java b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/model/eva/ClusteredVariantEntityIntegrationTest.java index 6ddf397f7..7d535412e 100644 --- a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/model/eva/ClusteredVariantEntityIntegrationTest.java +++ b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/model/eva/ClusteredVariantEntityIntegrationTest.java @@ -15,40 +15,36 @@ */ package uk.ac.ebi.eva.accession.core.model.eva; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; -import org.springframework.context.ApplicationContext; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.data.mongodb.core.query.Query; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.ampt2d.commons.accession.hashing.SHA1HashingFunction; - import uk.ac.ebi.eva.accession.core.configuration.nonhuman.MongoConfiguration; import uk.ac.ebi.eva.accession.core.model.ClusteredVariant; import uk.ac.ebi.eva.accession.core.model.IClusteredVariant; import uk.ac.ebi.eva.accession.core.summary.ClusteredVariantSummaryFunction; -import uk.ac.ebi.eva.accession.core.test.rule.FixSpringMongoDbRule; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.commons.core.models.VariantType; import java.util.Collections; import java.util.function.Function; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; -@RunWith(SpringRunner.class) +@ExtendWith(SpringExtension.class) @EnableAutoConfiguration -@ContextConfiguration(classes = {MongoConfiguration.class}) +@ContextConfiguration(classes = {MongoConfiguration.class, MongoTestConfiguration.class}) @TestPropertySource("classpath:test-model.properties") -public class ClusteredVariantEntityIntegrationTest { +public class ClusteredVariantEntityIntegrationTest extends MongoTestContainerHelper { private static final String TEST_DB = "test-db"; @@ -57,20 +53,13 @@ public class ClusteredVariantEntityIntegrationTest { @Autowired private MongoTemplate mongoTemplate; - //Required by nosql-unit - @Autowired - private ApplicationContext applicationContext; - - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName(TEST_DB).build()); - - @Before + @BeforeEach public void setUp() { + mongoTemplate.getDb().drop(); clusteredHashingFunction = new ClusteredVariantSummaryFunction().andThen(new SHA1HashingFunction()); } - @After + @AfterEach public void tearDown() { mongoTemplate.getDb().drop(); } @@ -105,7 +94,7 @@ private ClusteredVariantEntity createClusteredVariantEntity(Integer mapWeight) { ClusteredVariant variant = createClusteredVariant(); String hash = clusteredHashingFunction.apply(variant); ClusteredVariantEntity variantEntity = new ClusteredVariantEntity(3000000000L, hash, "asm", 1000, "contig", 100, - VariantType.SNV, false, null, 1, mapWeight); + VariantType.SNV, false, null, 1, mapWeight); return variantEntity; } diff --git a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/model/eva/ReleaseRecordSubmittedVariantEntityTest.java b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/model/eva/ReleaseRecordSubmittedVariantEntityTest.java index 061ef0708..c4ff019bf 100644 --- a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/model/eva/ReleaseRecordSubmittedVariantEntityTest.java +++ b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/model/eva/ReleaseRecordSubmittedVariantEntityTest.java @@ -1,9 +1,9 @@ package uk.ac.ebi.eva.accession.core.model.eva; -import org.junit.Test; +import org.junit.jupiter.api.Test; import uk.ac.ebi.eva.accession.core.model.ReleaseRecordSubmittedVariantEntity; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; public class ReleaseRecordSubmittedVariantEntityTest { diff --git a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/model/eva/SubmittedVariantEntityIntegrationTest.java b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/model/eva/SubmittedVariantEntityIntegrationTest.java index d45113261..92ed34acd 100644 --- a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/model/eva/SubmittedVariantEntityIntegrationTest.java +++ b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/model/eva/SubmittedVariantEntityIntegrationTest.java @@ -15,61 +15,47 @@ */ package uk.ac.ebi.eva.accession.core.model.eva; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; -import org.springframework.context.ApplicationContext; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.data.mongodb.core.query.Query; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.ampt2d.commons.accession.hashing.SHA1HashingFunction; - import uk.ac.ebi.eva.accession.core.configuration.nonhuman.MongoConfiguration; import uk.ac.ebi.eva.accession.core.model.ISubmittedVariant; import uk.ac.ebi.eva.accession.core.model.SubmittedVariant; import uk.ac.ebi.eva.accession.core.summary.SubmittedVariantSummaryFunction; -import uk.ac.ebi.eva.accession.core.test.rule.FixSpringMongoDbRule; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import java.util.Collections; import java.util.function.Function; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; -@RunWith(SpringRunner.class) +@ExtendWith(SpringExtension.class) @EnableAutoConfiguration -@ContextConfiguration(classes = {MongoConfiguration.class}) +@ContextConfiguration(classes = {MongoConfiguration.class, MongoTestConfiguration.class}) @TestPropertySource("classpath:test-model.properties") -public class SubmittedVariantEntityIntegrationTest { - - private static final String TEST_DB = "test-db"; - +public class SubmittedVariantEntityIntegrationTest extends MongoTestContainerHelper { private Function submittedHashingFunction; @Autowired private MongoTemplate mongoTemplate; - //Required by nosql-unit - @Autowired - private ApplicationContext applicationContext; - - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName(TEST_DB).build()); - - @Before + @BeforeEach public void setUp() { + mongoTemplate.getDb().drop(); submittedHashingFunction = new SubmittedVariantSummaryFunction().andThen(new SHA1HashingFunction()); } - @After + @AfterEach public void tearDown() { mongoTemplate.getDb().drop(); } @@ -104,8 +90,8 @@ private SubmittedVariantEntity createSubmittedVariantEntity(Integer mapWeight) { SubmittedVariant variant = createSubmittedVariant(); String hash = submittedHashingFunction.apply(variant); SubmittedVariantEntity variantEntity = new SubmittedVariantEntity(5000000000L, hash, "asm", 1000, "project", - "contig", 100, "A", "T", null, false, false, - false, false, 1, mapWeight); + "contig", 100, "A", "T", null, false, false, + false, false, 1, mapWeight); return variantEntity; } diff --git a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/model/eva/SubmittedVariantEntityTest.java b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/model/eva/SubmittedVariantEntityTest.java index 17ac67664..094d35a9f 100644 --- a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/model/eva/SubmittedVariantEntityTest.java +++ b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/model/eva/SubmittedVariantEntityTest.java @@ -15,15 +15,14 @@ */ package uk.ac.ebi.eva.accession.core.model.eva; -import org.junit.Test; - +import org.junit.jupiter.api.Test; import uk.ac.ebi.eva.accession.core.model.SubmittedVariant; -import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; import java.time.LocalDateTime; import java.time.Month; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; public class SubmittedVariantEntityTest { @@ -51,7 +50,7 @@ public class SubmittedVariantEntityTest { public void getModelWithFlagsTrue() { SubmittedVariant variant = new SubmittedVariant(ASSEMBLY_ACCESSION, TAXONOMY_ACCESSION, PROJECT_ACCESSION, CONTIG, START, - REF_A, ALT_T, CLUSTERED_VARIANT, true, true, true, true, null); + REF_A, ALT_T, CLUSTERED_VARIANT, true, true, true, true, null); SubmittedVariantEntity entity = new SubmittedVariantEntity(UNUSED_ACCESSION, UNUSED_HASHED_MESSAGE, variant, 1); assertEquals(variant, entity.getModel()); @@ -61,7 +60,7 @@ public void getModelWithFlagsTrue() { public void getModelWithFlagsFalse() { SubmittedVariant variant = new SubmittedVariant(ASSEMBLY_ACCESSION, TAXONOMY_ACCESSION, PROJECT_ACCESSION, CONTIG, START, - REF_A, ALT_T, CLUSTERED_VARIANT, false, false, false, false, null); + REF_A, ALT_T, CLUSTERED_VARIANT, false, false, false, false, null); SubmittedVariantEntity entity = new SubmittedVariantEntity(UNUSED_ACCESSION, UNUSED_HASHED_MESSAGE, variant, 1); assertEquals(variant, entity.getModel()); @@ -71,7 +70,7 @@ public void getModelWithFlagsFalse() { public void getModelWithDefaultFlags() { SubmittedVariant variant = new SubmittedVariant(ASSEMBLY_ACCESSION, TAXONOMY_ACCESSION, PROJECT_ACCESSION, CONTIG, START, - REF_A, ALT_T, CLUSTERED_VARIANT); + REF_A, ALT_T, CLUSTERED_VARIANT); SubmittedVariantEntity entity = new SubmittedVariantEntity(UNUSED_ACCESSION, UNUSED_HASHED_MESSAGE, variant, 1); variant.setSupportedByEvidence(true); @@ -81,11 +80,11 @@ public void getModelWithDefaultFlags() { assertEquals(variant, entity.getModel()); } - @Test(expected = IllegalArgumentException.class) + @Test public void flagsShouldNotBeNull() { - new SubmittedVariantEntity(UNUSED_ACCESSION, UNUSED_HASHED_MESSAGE, ASSEMBLY_ACCESSION, TAXONOMY_ACCESSION, - PROJECT_ACCESSION, CONTIG, START, REF_A, ALT_T, CLUSTERED_VARIANT, null, null, null, - null, 1); + assertThrows(IllegalArgumentException.class, () -> new SubmittedVariantEntity(UNUSED_ACCESSION, UNUSED_HASHED_MESSAGE, ASSEMBLY_ACCESSION, TAXONOMY_ACCESSION, + PROJECT_ACCESSION, CONTIG, START, REF_A, ALT_T, CLUSTERED_VARIANT, null, null, null, + null, 1)); } @Test @@ -94,7 +93,7 @@ public void creationDateIsReturnedInTheModel() { LocalDateTime createdDate = LocalDateTime.of(2018, Month.SEPTEMBER, 18, 9, 0); SubmittedVariant variant = new SubmittedVariant(ASSEMBLY_ACCESSION, TAXONOMY_ACCESSION, PROJECT_ACCESSION, CONTIG, START, - REF_A, ALT_T, CLUSTERED_VARIANT, false, false, false, false, createdDate); + REF_A, ALT_T, CLUSTERED_VARIANT, false, false, false, false, createdDate); SubmittedVariantEntity entity = new SubmittedVariantEntity(UNUSED_ACCESSION, UNUSED_HASHED_MESSAGE, variant, 1); assertEquals(createdDate, entity.getModel().getCreatedDate()); @@ -105,7 +104,7 @@ public void testChangeSVERefAltToUpperCase() { SubmittedVariant submittedVariant = new SubmittedVariant(ASSEMBLY_ACCESSION, TAXONOMY_ACCESSION, PROJECT_ACCESSION, CONTIG, START, "a", "t", CLUSTERED_VARIANT, false, false, false, false, LocalDateTime.of(2018, Month.SEPTEMBER, 18, 9, 0)); - SubmittedVariantEntity submittedVariantEntity = new SubmittedVariantEntity(123L, "hashedMessage", submittedVariant,1 ); + SubmittedVariantEntity submittedVariantEntity = new SubmittedVariantEntity(123L, "hashedMessage", submittedVariant, 1); assertEquals("A", submittedVariantEntity.getReferenceAllele()); assertEquals("T", submittedVariantEntity.getAlternateAllele()); } diff --git a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/mongoTemplate/MongoTemplateTest.java b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/mongoTemplate/MongoTemplateTest.java index 5c632e4ac..d90976752 100644 --- a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/mongoTemplate/MongoTemplateTest.java +++ b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/mongoTemplate/MongoTemplateTest.java @@ -16,33 +16,28 @@ package uk.ac.ebi.eva.accession.core.mongoTemplate; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.dao.DataAccessException; import org.springframework.data.mongodb.core.MongoTemplate; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.eva.commons.core.models.AnnotationMetadata; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.Mockito.doThrow; -@RunWith(SpringRunner.class) +@ExtendWith(SpringExtension.class) public class MongoTemplateTest { @MockBean private MongoTemplate mongoTemplate; - @Rule - public ExpectedException exceptionRule = ExpectedException.none(); - - @Test(expected = DataAccessException.class) + @Test public void testMongoTemplateWriteResultChecking() { AnnotationMetadata annotationMetadata = new AnnotationMetadata("vep_1", "vep_cache_1", true); - doThrow(new DataAccessException("Simulated exception") {}) - .when(mongoTemplate).save(annotationMetadata, "AnnotationMetadata"); + doThrow(new DataAccessException("Simulated exception") { + }).when(mongoTemplate).save(annotationMetadata, "AnnotationMetadata"); - mongoTemplate.save(annotationMetadata, "AnnotationMetadata"); + assertThrows(DataAccessException.class, () -> mongoTemplate.save(annotationMetadata, "AnnotationMetadata")); } } diff --git a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/repository/nonhuman/eva/ClusteredVariantAccessioningRepositoryTest.java b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/repository/nonhuman/eva/ClusteredVariantAccessioningRepositoryTest.java index 35e48013e..f4911b2e4 100644 --- a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/repository/nonhuman/eva/ClusteredVariantAccessioningRepositoryTest.java +++ b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/repository/nonhuman/eva/ClusteredVariantAccessioningRepositoryTest.java @@ -15,28 +15,23 @@ */ package uk.ac.ebi.eva.accession.core.repository.nonhuman.eva; -import com.lordofthejars.nosqlunit.annotation.UsingDataSet; -import com.lordofthejars.nosqlunit.core.LoadStrategyEnum; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.autoconfigure.orm.jpa.DataJpaTest; -import org.springframework.context.ApplicationContext; +import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.ampt2d.commons.accession.persistence.models.AccessionProjection; import uk.ac.ebi.ampt2d.commons.accession.persistence.mongodb.document.AccessionedDocument; - -import uk.ac.ebi.eva.accession.core.model.ClusteredVariant; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.ClusteredVariantAccessioningConfiguration; +import uk.ac.ebi.eva.accession.core.model.ClusteredVariant; import uk.ac.ebi.eva.accession.core.model.eva.ClusteredVariantEntity; import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; -import uk.ac.ebi.eva.accession.core.test.rule.FixSpringMongoDbRule; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.commons.core.models.VariantType; import java.time.LocalDateTime; @@ -47,13 +42,13 @@ import java.util.TreeSet; import java.util.stream.Collectors; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; -@RunWith(SpringRunner.class) +@ExtendWith(SpringExtension.class) @DataJpaTest @TestPropertySource("classpath:ss-accession-test.properties") @ContextConfiguration(classes = {ClusteredVariantAccessioningConfiguration.class, MongoTestConfiguration.class}) -public class ClusteredVariantAccessioningRepositoryTest { +public class ClusteredVariantAccessioningRepositoryTest extends MongoTestContainerHelper { private ClusteredVariant clusteredVariant; @@ -62,25 +57,26 @@ public class ClusteredVariantAccessioningRepositoryTest { @Autowired private ClusteredVariantAccessioningRepository repository; - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName("submitted-variants-test").build()); - - //Required by nosql-unit @Autowired - private ApplicationContext applicationContext; + private MongoTemplate mongoTemplate; - @Before + @BeforeEach public void setUp() { + mongoTemplate.getDb().drop(); clusteredVariant = new ClusteredVariant("GCA_000003055.3", 9913, "21", 20800319, VariantType.SNV, false, - LocalDateTime.of(2000, Month.SEPTEMBER, 19, 18, 2, 0)); + LocalDateTime.of(2000, Month.SEPTEMBER, 19, 18, 2, 0)); newClusteredVariant = new ClusteredVariant("assembly", 1111, "contig_2", 100, VariantType.SNV, false, - LocalDateTime.of(2000, Month.SEPTEMBER, 19, 18, 2, 0)); + LocalDateTime.of(2000, Month.SEPTEMBER, 19, 18, 2, 0)); } - @UsingDataSet(loadStrategy = LoadStrategyEnum.DELETE_ALL) + @AfterEach + void cleanDb() { + mongoTemplate.getDb().drop(); + } + + @Test public void queryAccessionRange() { long firstAccession = 1000L; @@ -92,25 +88,24 @@ public void queryAccessionRange() { repository.saveAll(variants); assertAccessionsEquals(Arrays.asList(firstAccession), - repository.findByAccessionGreaterThanEqualAndAccessionLessThanEqual(1000L, 1000L)); + repository.findByAccessionGreaterThanEqualAndAccessionLessThanEqual(1000L, 1000L)); assertAccessionsEquals(Arrays.asList(), - repository.findByAccessionGreaterThanEqualAndAccessionLessThanEqual(1001L, 1001L)); + repository.findByAccessionGreaterThanEqualAndAccessionLessThanEqual(1001L, 1001L)); assertAccessionsEquals(Arrays.asList(firstAccession, secondAccession), - repository.findByAccessionGreaterThanEqualAndAccessionLessThanEqual(1000L, 1005L)); + repository.findByAccessionGreaterThanEqualAndAccessionLessThanEqual(1000L, 1005L)); assertAccessionsEquals(Arrays.asList(secondAccession), - repository.findByAccessionGreaterThanEqualAndAccessionLessThanEqual(1001L, 1005L)); + repository.findByAccessionGreaterThanEqualAndAccessionLessThanEqual(1001L, 1005L)); } private void assertAccessionsEquals(List expectedAccessions, List> accessionsProjection) { assertEquals(new TreeSet<>(expectedAccessions), - accessionsProjection.stream().map(AccessionProjection::getAccession).collect(Collectors.toSet())); + accessionsProjection.stream().map(AccessionProjection::getAccession).collect(Collectors.toSet())); } - @UsingDataSet(loadStrategy = LoadStrategyEnum.DELETE_ALL) @Test public void testQueryAssemblyAndAccessionFilter() { long firstAccession = 1000L; @@ -121,12 +116,12 @@ public void testQueryAssemblyAndAccessionFilter() { repository.saveAll(variants); - List accessions = repository.findByAssemblyAccessionAndAccessionIn( + List accessions = repository.findByAssemblyAccessionAndAccessionIn( "assembly", Arrays.asList(firstAccession, secondAccession) ).stream().map(AccessionedDocument::getAccession).collect(Collectors.toList()); assertEquals(1, accessions.size()); - accessions = repository.findByAssemblyAccessionAndAccessionIn( + accessions = repository.findByAssemblyAccessionAndAccessionIn( "assembly", Collections.singletonList(firstAccession) ).stream().map(AccessionedDocument::getAccession).collect(Collectors.toList()); assertEquals(0, accessions.size()); diff --git a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/repository/nonhuman/eva/SubmittedVariantAccessioningRepositoryTest.java b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/repository/nonhuman/eva/SubmittedVariantAccessioningRepositoryTest.java index b27dd3d71..bb6d40b05 100644 --- a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/repository/nonhuman/eva/SubmittedVariantAccessioningRepositoryTest.java +++ b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/repository/nonhuman/eva/SubmittedVariantAccessioningRepositoryTest.java @@ -15,29 +15,23 @@ */ package uk.ac.ebi.eva.accession.core.repository.nonhuman.eva; -import com.lordofthejars.nosqlunit.annotation.UsingDataSet; -import com.lordofthejars.nosqlunit.core.LoadStrategyEnum; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.autoconfigure.orm.jpa.DataJpaTest; -import org.springframework.context.ApplicationContext; +import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.ampt2d.commons.accession.persistence.models.AccessionProjection; import uk.ac.ebi.ampt2d.commons.accession.persistence.mongodb.document.AccessionedDocument; - -import uk.ac.ebi.eva.accession.core.model.SubmittedVariant; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.SubmittedVariantAccessioningConfiguration; +import uk.ac.ebi.eva.accession.core.model.SubmittedVariant; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; +import uk.ac.ebi.eva.accession.core.test.configuration.JPATestConfiguration; import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; -import uk.ac.ebi.eva.accession.core.test.rule.FixSpringMongoDbRule; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import java.util.Arrays; import java.util.Collections; @@ -45,16 +39,16 @@ import java.util.TreeSet; import java.util.stream.Collectors; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import static uk.ac.ebi.eva.accession.core.model.ISubmittedVariant.DEFAULT_ASSEMBLY_MATCH; import static uk.ac.ebi.eva.accession.core.model.ISubmittedVariant.DEFAULT_SUPPORTED_BY_EVIDENCE; import static uk.ac.ebi.eva.accession.core.model.ISubmittedVariant.DEFAULT_VALIDATED; -@RunWith(SpringRunner.class) -@DataJpaTest +@ExtendWith(SpringExtension.class) @TestPropertySource("classpath:ss-accession-test.properties") -@ContextConfiguration(classes = {SubmittedVariantAccessioningConfiguration.class, MongoTestConfiguration.class}) -public class SubmittedVariantAccessioningRepositoryTest { +@ContextConfiguration(classes = {SubmittedVariantAccessioningConfiguration.class, MongoTestConfiguration.class, + JPATestConfiguration.class}) +public class SubmittedVariantAccessioningRepositoryTest extends MongoTestContainerHelper { private static final Long CLUSTERED_VARIANT = null; @@ -67,32 +61,30 @@ public class SubmittedVariantAccessioningRepositoryTest { @Autowired private SubmittedVariantAccessioningRepository repository; - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName("submitted-variants-test").build()); - - //Required by nosql-unit @Autowired - private ApplicationContext applicationContext; + private MongoTemplate mongoTemplate; - @Rule - public ExpectedException expectedException = ExpectedException.none(); - - @Before + @BeforeEach public void setUp() { + mongoTemplate.getDb().drop(); submittedVariant = new SubmittedVariant("GCA_000003055.3", 9913, PROJECT, "21", 20800319, "C", "T", - CLUSTERED_VARIANT, DEFAULT_SUPPORTED_BY_EVIDENCE, - DEFAULT_ASSEMBLY_MATCH, true, - DEFAULT_VALIDATED, null); + CLUSTERED_VARIANT, DEFAULT_SUPPORTED_BY_EVIDENCE, + DEFAULT_ASSEMBLY_MATCH, true, + DEFAULT_VALIDATED, null); newSubmittedVariant = new SubmittedVariant("assembly", 1111, "project", "contig_2", 100, "ref", "alt", - CLUSTERED_VARIANT, DEFAULT_SUPPORTED_BY_EVIDENCE, - DEFAULT_ASSEMBLY_MATCH, true, - DEFAULT_VALIDATED, null); + CLUSTERED_VARIANT, DEFAULT_SUPPORTED_BY_EVIDENCE, + DEFAULT_ASSEMBLY_MATCH, true, + DEFAULT_VALIDATED, null); + + } + @AfterEach + void cleanDb() { + mongoTemplate.getDb().drop(); } - @UsingDataSet(loadStrategy = LoadStrategyEnum.DELETE_ALL) + @Test public void queryAccessionRange() { long firstAccession = 1000L; @@ -104,25 +96,24 @@ public void queryAccessionRange() { repository.saveAll(variants); assertAccessionsEquals(Arrays.asList(firstAccession), - repository.findByAccessionGreaterThanEqualAndAccessionLessThanEqual(1000L, 1000L)); + repository.findByAccessionGreaterThanEqualAndAccessionLessThanEqual(1000L, 1000L)); assertAccessionsEquals(Arrays.asList(), - repository.findByAccessionGreaterThanEqualAndAccessionLessThanEqual(1001L, 1001L)); + repository.findByAccessionGreaterThanEqualAndAccessionLessThanEqual(1001L, 1001L)); assertAccessionsEquals(Arrays.asList(firstAccession, secondAccession), - repository.findByAccessionGreaterThanEqualAndAccessionLessThanEqual(1000L, 1005L)); + repository.findByAccessionGreaterThanEqualAndAccessionLessThanEqual(1000L, 1005L)); assertAccessionsEquals(Arrays.asList(secondAccession), - repository.findByAccessionGreaterThanEqualAndAccessionLessThanEqual(1001L, 1005L)); + repository.findByAccessionGreaterThanEqualAndAccessionLessThanEqual(1001L, 1005L)); } private void assertAccessionsEquals(List expectedAccessions, List> accessionsProjection) { assertEquals(new TreeSet<>(expectedAccessions), - accessionsProjection.stream().map(AccessionProjection::getAccession).collect(Collectors.toSet())); + accessionsProjection.stream().map(AccessionProjection::getAccession).collect(Collectors.toSet())); } - @UsingDataSet(loadStrategy = LoadStrategyEnum.DELETE_ALL) @Test public void testQueryAssemblyAndAccessionFilter() { long firstAccession = 1000L; @@ -132,12 +123,12 @@ public void testQueryAssemblyAndAccessionFilter() { new SubmittedVariantEntity(secondAccession, "hash-2", newSubmittedVariant, 1)); repository.saveAll(variants); - List accessions = repository.findByReferenceSequenceAccessionAndAccessionIn( + List accessions = repository.findByReferenceSequenceAccessionAndAccessionIn( "assembly", Arrays.asList(firstAccession, secondAccession) ).stream().map(AccessionedDocument::getAccession).collect(Collectors.toList()); assertEquals(1, accessions.size()); - accessions = repository.findByReferenceSequenceAccessionAndAccessionIn( + accessions = repository.findByReferenceSequenceAccessionAndAccessionIn( "assembly", Collections.singletonList(firstAccession) ).stream().map(AccessionedDocument::getAccession).collect(Collectors.toList()); assertEquals(0, accessions.size()); diff --git a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/service/human/dbsnp/HumanDbsnpClusteredVariantAccessioningServiceTest.java b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/service/human/dbsnp/HumanDbsnpClusteredVariantAccessioningServiceTest.java index 642f2f151..30682e090 100644 --- a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/service/human/dbsnp/HumanDbsnpClusteredVariantAccessioningServiceTest.java +++ b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/service/human/dbsnp/HumanDbsnpClusteredVariantAccessioningServiceTest.java @@ -15,26 +15,25 @@ */ package uk.ac.ebi.eva.accession.core.service.human.dbsnp; -import com.lordofthejars.nosqlunit.annotation.UsingDataSet; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; -import org.junit.Ignore; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.boot.test.autoconfigure.orm.jpa.DataJpaTest; -import org.springframework.context.ApplicationContext; +import org.springframework.core.io.ResourceLoader; +import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.ampt2d.commons.accession.core.models.AccessionWrapper; - import uk.ac.ebi.eva.accession.core.model.ClusteredVariant; import uk.ac.ebi.eva.accession.core.model.IClusteredVariant; +import uk.ac.ebi.eva.accession.core.test.configuration.JPATestConfiguration; import uk.ac.ebi.eva.accession.core.test.configuration.human.MongoHumanTestConfiguration; -import uk.ac.ebi.eva.accession.core.test.rule.FixSpringMongoDbRule; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; +import uk.ac.ebi.eva.accession.core.utils.MongoTestDataLoader; import uk.ac.ebi.eva.commons.core.models.VariantType; import java.time.LocalDateTime; @@ -42,14 +41,12 @@ import java.util.Collections; import java.util.List; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; -@RunWith(SpringRunner.class) -@DataJpaTest +@ExtendWith(SpringExtension.class) @TestPropertySource("classpath:ss-accession-test.properties") -@UsingDataSet(locations = {"/test-data/dbsnpClusteredVariantEntity.json", "/test-data/dbsnpClusteredVariantOperationEntity.json"}) -@ContextConfiguration(classes = {MongoHumanTestConfiguration.class}) -public class HumanDbsnpClusteredVariantAccessioningServiceTest { +@ContextConfiguration(classes = {MongoHumanTestConfiguration.class, JPATestConfiguration.class}) +public class HumanDbsnpClusteredVariantAccessioningServiceTest extends MongoTestContainerHelper { private static final Long HUMAN_ACTIVE_RS_ID_1 = 1118L; @@ -64,16 +61,27 @@ public class HumanDbsnpClusteredVariantAccessioningServiceTest { "CM000663.2", 72348112L, VariantType.INDEL, false, LocalDateTime.of(2017, Month.NOVEMBER, 11, 9, 55, 0)); @Autowired - @Qualifier("humanService") - private HumanDbsnpClusteredVariantAccessioningService humanService; + private MongoTemplate mongoTemplate; - //Required by nosql-unit @Autowired - private ApplicationContext applicationContext; + private ResourceLoader resourceLoader; + + @BeforeEach + public void setUp() { + mongoTemplate.getDb().drop(); + MongoTestDataLoader mongoTestDataLoader = new MongoTestDataLoader(mongoTemplate, resourceLoader); + mongoTestDataLoader.load("/test-data/dbsnpClusteredVariantEntity.json"); + mongoTestDataLoader.load("/test-data/dbsnpClusteredVariantOperationEntity.json"); + } - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName("human-variants-test").build()); + @AfterEach + public void tearDown() { + mongoTemplate.getDb().drop(); + } + + @Autowired + @Qualifier("humanService") + private HumanDbsnpClusteredVariantAccessioningService humanService; @Test public void getHumanActiveVariant() { @@ -85,7 +93,7 @@ public void getHumanActiveVariant() { } @Test - @Ignore("humanService.getAllByAccession is not returning multiple variants yet") + @Disabled("humanService.getAllByAccession is not returning multiple variants yet") public void getHumanActiveMultipleVariants() { List> clusteredVariants = humanService.getAllByAccession(HUMAN_ACTIVE_RS_ID_2); diff --git a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/service/nonhuman/ClusteredVariantAccessioningServiceTest.java b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/service/nonhuman/ClusteredVariantAccessioningServiceTest.java index 637596ebb..ed9324c80 100644 --- a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/service/nonhuman/ClusteredVariantAccessioningServiceTest.java +++ b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/service/nonhuman/ClusteredVariantAccessioningServiceTest.java @@ -15,28 +15,23 @@ */ package uk.ac.ebi.eva.accession.core.service.nonhuman; -import com.lordofthejars.nosqlunit.annotation.UsingDataSet; -import com.lordofthejars.nosqlunit.core.LoadStrategyEnum; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; -import org.junit.After; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.autoconfigure.orm.jpa.DataJpaTest; -import org.springframework.context.ApplicationContext; +import org.springframework.core.io.ResourceLoader; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.data.mongodb.core.query.Query; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.ampt2d.commons.accession.core.exceptions.AccessionDeprecatedException; import uk.ac.ebi.ampt2d.commons.accession.core.exceptions.AccessionDoesNotExistException; import uk.ac.ebi.ampt2d.commons.accession.core.exceptions.AccessionMergedException; import uk.ac.ebi.ampt2d.commons.accession.core.models.GetOrCreateAccessionWrapper; import uk.ac.ebi.ampt2d.commons.accession.hashing.SHA1HashingFunction; - import uk.ac.ebi.eva.accession.core.configuration.nonhuman.ClusteredVariantAccessioningConfiguration; import uk.ac.ebi.eva.accession.core.model.ClusteredVariant; import uk.ac.ebi.eva.accession.core.model.IClusteredVariant; @@ -44,7 +39,8 @@ import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpClusteredVariantOperationEntity; import uk.ac.ebi.eva.accession.core.summary.ClusteredVariantSummaryFunction; import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; -import uk.ac.ebi.eva.accession.core.test.rule.FixSpringMongoDbRule; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; +import uk.ac.ebi.eva.accession.core.utils.MongoTestDataLoader; import uk.ac.ebi.eva.commons.core.models.VariantType; import java.util.Arrays; @@ -52,26 +48,21 @@ import java.util.List; import java.util.function.Function; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; -@RunWith(SpringRunner.class) +@ExtendWith(SpringExtension.class) @DataJpaTest @TestPropertySource("classpath:rs-accession-test.properties") @ContextConfiguration(classes = {ClusteredVariantAccessioningConfiguration.class, MongoTestConfiguration.class}) -public class ClusteredVariantAccessioningServiceTest { +public class ClusteredVariantAccessioningServiceTest extends MongoTestContainerHelper { private static String TEST_APPLICATION_INSTANCE_ID = "test-application-instance-id"; - //Required by nosql-unit - @Autowired - private ApplicationContext applicationContext; - @Autowired private MongoTemplate mongoTemplate; - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName("clustered-variants-test").build()); + @Autowired + private ResourceLoader resourceLoader; @Autowired private ClusteredVariantAccessioningService service; @@ -79,13 +70,16 @@ public class ClusteredVariantAccessioningServiceTest { private Function clusteredHashingFunction = new ClusteredVariantSummaryFunction().andThen(new SHA1HashingFunction()); - @After + @BeforeEach + public void setUp() { + mongoTemplate.getDb().drop(); + } + + @AfterEach public void tearDown() throws Exception { - mongoTemplate.dropCollection(DbsnpClusteredVariantEntity.class); - mongoTemplate.dropCollection(DbsnpClusteredVariantOperationEntity.class); + mongoTemplate.getDb().drop(); } - @UsingDataSet(loadStrategy = LoadStrategyEnum.DELETE_ALL) @Test public void sameAccessionsAreReturnedForIdenticalVariants() throws Exception { List variants = Arrays.asList( @@ -121,9 +115,10 @@ public void mergeRemappedVariants() throws AccessionMergedException, AccessionDo assertEquals(1, mongoTemplate.count(new Query(), DbsnpClusteredVariantOperationEntity.class)); } - @UsingDataSet(locations = {"/test-data/dbsnpClusteredVariantEntity.json"}) @Test public void getAllByAccession() throws AccessionMergedException, AccessionDoesNotExistException, AccessionDeprecatedException { + new MongoTestDataLoader(mongoTemplate, resourceLoader).load("/test-data/dbsnpClusteredVariantEntity.json"); + assertEquals(2, service.getAllByAccession(1314L).size()); } } diff --git a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/service/nonhuman/SubmittedVariantAccessioningServiceTest.java b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/service/nonhuman/SubmittedVariantAccessioningServiceTest.java index 4c888303d..9a1b6c91a 100644 --- a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/service/nonhuman/SubmittedVariantAccessioningServiceTest.java +++ b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/service/nonhuman/SubmittedVariantAccessioningServiceTest.java @@ -15,24 +15,19 @@ */ package uk.ac.ebi.eva.accession.core.service.nonhuman; -import com.lordofthejars.nosqlunit.annotation.UsingDataSet; -import com.lordofthejars.nosqlunit.core.LoadStrategyEnum; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; import com.mongodb.client.MongoCursor; import org.bson.Document; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.autoconfigure.orm.jpa.DataJpaTest; -import org.springframework.context.ApplicationContext; -import org.springframework.data.mongodb.MongoDbFactory; +import org.springframework.core.io.ResourceLoader; +import org.springframework.data.mongodb.MongoDatabaseFactory; +import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.ampt2d.commons.accession.core.AccessionSaveMode; import uk.ac.ebi.ampt2d.commons.accession.core.exceptions.AccessionCouldNotBeGeneratedException; import uk.ac.ebi.ampt2d.commons.accession.core.exceptions.AccessionDeprecatedException; @@ -44,7 +39,6 @@ import uk.ac.ebi.ampt2d.commons.accession.core.models.GetOrCreateAccessionWrapper; import uk.ac.ebi.ampt2d.commons.accession.core.models.IEvent; import uk.ac.ebi.ampt2d.commons.accession.hashing.SHA1HashingFunction; - import uk.ac.ebi.eva.accession.core.configuration.nonhuman.SubmittedVariantAccessioningConfiguration; import uk.ac.ebi.eva.accession.core.generators.DbsnpMonotonicAccessionGenerator; import uk.ac.ebi.eva.accession.core.model.ISubmittedVariant; @@ -54,8 +48,10 @@ import uk.ac.ebi.eva.accession.core.service.nonhuman.dbsnp.DbsnpSubmittedVariantMonotonicAccessioningService; import uk.ac.ebi.eva.accession.core.service.nonhuman.eva.SubmittedVariantInactiveService; import uk.ac.ebi.eva.accession.core.summary.SubmittedVariantSummaryFunction; +import uk.ac.ebi.eva.accession.core.test.configuration.JPATestConfiguration; import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; -import uk.ac.ebi.eva.accession.core.test.rule.FixSpringMongoDbRule; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; +import uk.ac.ebi.eva.accession.core.utils.MongoTestDataLoader; import uk.ac.ebi.eva.commons.core.models.contigalias.ContigNamingConvention; import java.time.LocalDateTime; @@ -68,19 +64,20 @@ import java.util.NoSuchElementException; import java.util.stream.Collectors; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; import static uk.ac.ebi.eva.accession.core.model.ISubmittedVariant.DEFAULT_ALLELES_MATCH; import static uk.ac.ebi.eva.accession.core.model.ISubmittedVariant.DEFAULT_ASSEMBLY_MATCH; import static uk.ac.ebi.eva.accession.core.model.ISubmittedVariant.DEFAULT_SUPPORTED_BY_EVIDENCE; import static uk.ac.ebi.eva.accession.core.model.ISubmittedVariant.DEFAULT_VALIDATED; -@RunWith(SpringRunner.class) -@DataJpaTest +@ExtendWith(SpringExtension.class) @TestPropertySource("classpath:ss-accession-test.properties") -@ContextConfiguration(classes = {SubmittedVariantAccessioningConfiguration.class, MongoTestConfiguration.class}) -public class SubmittedVariantAccessioningServiceTest { +@ContextConfiguration(classes = {SubmittedVariantAccessioningConfiguration.class, MongoTestConfiguration.class, + JPATestConfiguration.class}) +public class SubmittedVariantAccessioningServiceTest extends MongoTestContainerHelper { private static String TEST_APPLICATION_INSTANCE_ID = "test-application-instance-id"; private static final Long CLUSTERED_VARIANT = null; @@ -121,10 +118,6 @@ public class SubmittedVariantAccessioningServiceTest { @Autowired private DbsnpSubmittedVariantAccessioningDatabaseService dbServiceDbsnp; - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName("submitted-variants-test").build()); - @Autowired private SubmittedVariantAccessioningService service; @@ -137,47 +130,49 @@ public class SubmittedVariantAccessioningServiceTest { @Autowired private Long accessioningMonotonicInitSs; - //Required by nosql-unit @Autowired - private ApplicationContext applicationContext; - - @Rule - public ExpectedException expectedException = ExpectedException.none(); + private MongoDatabaseFactory mongoDbFactory; @Autowired - private MongoDbFactory mongoDbFactory; + private ResourceLoader resourceLoader; + @Autowired + private MongoTemplate mongoTemplate; - @Before + @BeforeEach public void setUp() { + mongoTemplate.getDb().drop(); submittedVariant = new SubmittedVariant("GCA_000003055.3", 9913, PROJECT, "21", 20800319, "C", "T", - CLUSTERED_VARIANT, DEFAULT_SUPPORTED_BY_EVIDENCE, - DEFAULT_ASSEMBLY_MATCH, true, - DEFAULT_VALIDATED, null); + CLUSTERED_VARIANT, DEFAULT_SUPPORTED_BY_EVIDENCE, + DEFAULT_ASSEMBLY_MATCH, true, + DEFAULT_VALIDATED, null); newSubmittedVariant = new SubmittedVariant("assembly", 1111, "project", "contig_2", 100, "ref", "alt", - CLUSTERED_VARIANT, DEFAULT_SUPPORTED_BY_EVIDENCE, - DEFAULT_ASSEMBLY_MATCH, true, - DEFAULT_VALIDATED, null); + CLUSTERED_VARIANT, DEFAULT_SUPPORTED_BY_EVIDENCE, + DEFAULT_ASSEMBLY_MATCH, true, + DEFAULT_VALIDATED, null); dbsnpSubmittedVariant = new SubmittedVariant("GCA_000009999.3", 9999, PROJECT_DBSNP, "21", 20849999, "", "GG", - CLUSTERED_VARIANT, DEFAULT_SUPPORTED_BY_EVIDENCE, - DEFAULT_ASSEMBLY_MATCH, true, - DEFAULT_VALIDATED, null); + CLUSTERED_VARIANT, DEFAULT_SUPPORTED_BY_EVIDENCE, + DEFAULT_ASSEMBLY_MATCH, true, + DEFAULT_VALIDATED, null); submittedVariantModified = new SubmittedVariant("GCA_000003055.3", 9913, PROJECT, "21", 20800319, - "C", "TCTC", CLUSTERED_VARIANT, DEFAULT_SUPPORTED_BY_EVIDENCE, - DEFAULT_ASSEMBLY_MATCH, true, DEFAULT_VALIDATED, null); -// submittedVariantModified.setCreatedDate(LocalDateTime.now()); + "C", "TCTC", CLUSTERED_VARIANT, DEFAULT_SUPPORTED_BY_EVIDENCE, + DEFAULT_ASSEMBLY_MATCH, true, DEFAULT_VALIDATED, null); + } + + @AfterEach + public void tearDown() { + mongoTemplate.getDb().drop(); } - @UsingDataSet(loadStrategy = LoadStrategyEnum.DELETE_ALL) @Test public void sameAccessionsAreReturnedForIdenticalVariants() throws AccessionCouldNotBeGeneratedException { List variants = Arrays.asList( new SubmittedVariant("assembly", 1111, "project", "contig_1", 100, "ref", "alt", CLUSTERED_VARIANT, - DEFAULT_SUPPORTED_BY_EVIDENCE, DEFAULT_ASSEMBLY_MATCH, ALLELES_MATCH, - DEFAULT_VALIDATED, null), + DEFAULT_SUPPORTED_BY_EVIDENCE, DEFAULT_ASSEMBLY_MATCH, ALLELES_MATCH, + DEFAULT_VALIDATED, null), newSubmittedVariant); List> generatedAccessions = service.getOrCreate(variants, TEST_APPLICATION_INSTANCE_ID); @@ -186,13 +181,12 @@ public void sameAccessionsAreReturnedForIdenticalVariants() throws AccessionCoul assertEquals(new HashSet<>(generatedAccessions), new HashSet<>(retrievedAccessions)); } - @UsingDataSet(loadStrategy = LoadStrategyEnum.DELETE_ALL) @Test public void sameAccessionsAreReturnedForEquivalentVariants() throws AccessionCouldNotBeGeneratedException { List originalVariants = Arrays.asList( new SubmittedVariant("assembly", 1111, "project", "contig_1", 100, "ref", "alt", 10L, - DEFAULT_SUPPORTED_BY_EVIDENCE, DEFAULT_ASSEMBLY_MATCH, ALLELES_MATCH, - DEFAULT_VALIDATED, null), + DEFAULT_SUPPORTED_BY_EVIDENCE, DEFAULT_ASSEMBLY_MATCH, ALLELES_MATCH, + DEFAULT_VALIDATED, null), newSubmittedVariant); List requestedVariants = Arrays.asList( new SubmittedVariant("assembly", 1111, "project", "contig_1", 100, "ref", "alt", null), @@ -205,75 +199,86 @@ public void sameAccessionsAreReturnedForEquivalentVariants() throws AccessionCou assertEquals(new HashSet<>(generatedAccessions), new HashSet<>(retrievedAccessions)); } - @UsingDataSet(locations = {"/test-data/submittedVariantEntity.json", "/test-data/dbsnpSubmittedVariantEntity.json"}) @Test public void getOrCreateAccessionsInBothRepositories() throws AccessionCouldNotBeGeneratedException { + MongoTestDataLoader mongoTestDataLoader = new MongoTestDataLoader(mongoTemplate, resourceLoader); + mongoTestDataLoader.load("/test-data/submittedVariantEntity.json"); + mongoTestDataLoader.load("/test-data/dbsnpSubmittedVariantEntity.json"); + List variants = Arrays.asList(submittedVariant, newSubmittedVariant, dbsnpSubmittedVariant); List> submittedVariants = service.getOrCreate(variants, TEST_APPLICATION_INSTANCE_ID); long accession = submittedVariants.stream() - .filter(x -> x.getData().getProjectAccession().equals(PROJECT)) - .findAny().orElseThrow(NoSuchElementException::new) - .getAccession(); + .filter(x -> x.getData().getProjectAccession().equals(PROJECT)) + .findAny().orElseThrow(NoSuchElementException::new) + .getAccession(); assertEquals(ACCESSION, accession); long accessionDbsnp = submittedVariants.stream() - .filter(x -> x.getData().getProjectAccession().equals(PROJECT_DBSNP)) - .findAny().orElseThrow(NoSuchElementException::new) - .getAccession(); + .filter(x -> x.getData().getProjectAccession().equals(PROJECT_DBSNP)) + .findAny().orElseThrow(NoSuchElementException::new) + .getAccession(); assertEquals(ACCESSION_DBSNP_1, accessionDbsnp); assertEquals(3, submittedVariants.size()); } - @UsingDataSet(locations = {"/test-data/dbsnpSubmittedVariantEntity.json"}) @Test public void getOrCreateDbsnpIdenticalVariants() throws AccessionCouldNotBeGeneratedException { + new MongoTestDataLoader(mongoTemplate, resourceLoader).load("/test-data/dbsnpSubmittedVariantEntity.json"); + SubmittedVariant variantPresentInDbsnp = new SubmittedVariant("GCA_000009999.3", 9999, PROJECT_DBSNP, "21", - 20849999, "", "GG", null); + 20849999, "", "GG", null); List> generatedAccessions = service.getOrCreate( Collections.singletonList(variantPresentInDbsnp), TEST_APPLICATION_INSTANCE_ID); assertEquals(Collections.singleton(ACCESSION_DBSNP_1), - generatedAccessions.stream().map(AccessionWrapper::getAccession).collect(Collectors.toSet())); + generatedAccessions.stream().map(AccessionWrapper::getAccession).collect(Collectors.toSet())); } - @UsingDataSet(locations = {"/test-data/dbsnpSubmittedVariantEntity.json"}) @Test public void getOrCreateDbsnpEquivalentVariants() throws AccessionCouldNotBeGeneratedException { + new MongoTestDataLoader(mongoTemplate, resourceLoader).load("/test-data/dbsnpSubmittedVariantEntity.json"); + SubmittedVariant identicalVariantPresentInDbsnp = new SubmittedVariant("GCA_000009999.3", 9999, PROJECT_DBSNP, - "21", 20849999, "", "GG", null); + "21", 20849999, "", "GG", null); List> generatedAccessions = service.getOrCreate( Arrays.asList(identicalVariantPresentInDbsnp), TEST_APPLICATION_INSTANCE_ID); assertEquals(Collections.singleton(ACCESSION_DBSNP_1), - generatedAccessions.stream().map(AccessionWrapper::getAccession).collect(Collectors.toSet())); + generatedAccessions.stream().map(AccessionWrapper::getAccession).collect(Collectors.toSet())); int differentTaxonomy = 1000; SubmittedVariant equivalentVariantPresentInDbsnp = new SubmittedVariant("GCA_000009999.3", differentTaxonomy, - PROJECT_DBSNP, "21", 20849999, "", "GG", - null); + PROJECT_DBSNP, "21", 20849999, "", "GG", + null); List> generatedAccessions2 = service.getOrCreate( Arrays.asList(identicalVariantPresentInDbsnp, equivalentVariantPresentInDbsnp), TEST_APPLICATION_INSTANCE_ID); assertEquals(Collections.singleton(ACCESSION_DBSNP_1), - generatedAccessions2.stream().map(AccessionWrapper::getAccession).collect(Collectors.toSet())); + generatedAccessions2.stream().map(AccessionWrapper::getAccession).collect(Collectors.toSet())); } - @UsingDataSet(locations = {"/test-data/submittedVariantEntity.json", "/test-data/dbsnpSubmittedVariantEntity.json"}) @Test public void getFromBothRepositories() { + MongoTestDataLoader mongoTestDataLoader = new MongoTestDataLoader(mongoTemplate, resourceLoader); + mongoTestDataLoader.load("/test-data/submittedVariantEntity.json"); + mongoTestDataLoader.load("/test-data/dbsnpSubmittedVariantEntity.json"); + List variants = Arrays.asList(submittedVariant, newSubmittedVariant, dbsnpSubmittedVariant); List> accessions = service.get(variants); assertEquals(2, accessions.size()); } - @UsingDataSet(locations = {"/test-data/submittedVariantEntity.json", "/test-data/dbsnpSubmittedVariantEntity.json"}) @Test public void getByAccessionsFromBothRepositories() throws AccessionMergedException, AccessionDoesNotExistException, AccessionDeprecatedException { + MongoTestDataLoader mongoTestDataLoader = new MongoTestDataLoader(mongoTemplate, resourceLoader); + mongoTestDataLoader.load("/test-data/submittedVariantEntity.json"); + mongoTestDataLoader.load("/test-data/dbsnpSubmittedVariantEntity.json"); + AccessionWrapper retrievedSubmittedVariant = service.getByAccession(ACCESSION); assertEquals(submittedVariant, retrievedSubmittedVariant.getData()); AccessionWrapper retrievedSubmittedDbsnpVariant = service.getByAccession( @@ -281,23 +286,28 @@ public void getByAccessionsFromBothRepositories() assertEquals(dbsnpSubmittedVariant, retrievedSubmittedDbsnpVariant.getData()); } - @UsingDataSet(locations = {"/test-data/dbsnpSubmittedVariantEntity.json"}) @Test public void getAllByAccession() throws AccessionMergedException, AccessionDoesNotExistException, AccessionDeprecatedException { + new MongoTestDataLoader(mongoTemplate, resourceLoader).load("/test-data/dbsnpSubmittedVariantEntity.json"); + assertEquals(2, service.getAllByAccession(2200000002L).size()); } - @UsingDataSet(locations = {"/test-data/dbsnpSubmittedVariantEntity.json"}) @Test public void getAllByIdFields() { + new MongoTestDataLoader(mongoTemplate, resourceLoader).load("/test-data/dbsnpSubmittedVariantEntity.json"); + assertEquals(2, service.getAllByIdFields("GCA_000009999.3", "21", Arrays.asList("DBSNP999", "DBSNP111"), - 20849999L, "", "GG", ContigNamingConvention.INSDC).size()); + 20849999L, "", "GG", ContigNamingConvention.INSDC).size()); } - @UsingDataSet(locations = {"/test-data/submittedVariantEntity.json", "/test-data/dbsnpSubmittedVariantEntity.json"}) @Test public void getByAccessionAndVersionFromBothRepositories() throws AccessionMergedException, AccessionDoesNotExistException, AccessionDeprecatedException { + MongoTestDataLoader mongoTestDataLoader = new MongoTestDataLoader(mongoTemplate, resourceLoader); + mongoTestDataLoader.load("/test-data/submittedVariantEntity.json"); + mongoTestDataLoader.load("/test-data/dbsnpSubmittedVariantEntity.json"); + AccessionWrapper retrievedSubmittedVariant = service.getByAccessionAndVersion( ACCESSION, 1); assertEquals(submittedVariant, retrievedSubmittedVariant.getData()); @@ -307,10 +317,11 @@ public void getByAccessionAndVersionFromBothRepositories() throws AccessionMerge assertEquals(dbsnpSubmittedVariant, retrievedDbsnpSubmittedVariant.getData()); } - @UsingDataSet(locations = {"/test-data/submittedVariantEntity.json"}) @Test public void updateSubmittedVariant() throws AccessionDeprecatedException, AccessionDoesNotExistException, AccessionMergedException, HashAlreadyExistsException { + new MongoTestDataLoader(mongoTemplate, resourceLoader).load("/test-data/submittedVariantEntity.json"); + service.update(ACCESSION, 1, submittedVariantModified); assertVariantUpdated(ACCESSION, submittedVariantModified); } @@ -330,18 +341,20 @@ private void assertVariantUpdated(long accession, ISubmittedVariant modifiedVari assertNull(lastOperation.getMergedInto()); } - @UsingDataSet(locations = {"/test-data/dbsnpSubmittedVariantEntity.json"}) @Test public void updateDbsnpSubmittedVariant() throws AccessionDeprecatedException, AccessionDoesNotExistException, AccessionMergedException, HashAlreadyExistsException { + new MongoTestDataLoader(mongoTemplate, resourceLoader).load("/test-data/dbsnpSubmittedVariantEntity.json"); + service.update(ACCESSION_DBSNP_1, 1, submittedVariantModified); assertVariantUpdated(ACCESSION_DBSNP_1, submittedVariantModified); } - @UsingDataSet(locations = {"/test-data/submittedVariantEntity.json"}) @Test public void patchSubmittedVariant() throws AccessionDeprecatedException, AccessionDoesNotExistException, AccessionMergedException, HashAlreadyExistsException { + new MongoTestDataLoader(mongoTemplate, resourceLoader).load("/test-data/submittedVariantEntity.json"); + service.patch(ACCESSION, submittedVariantModified); assertVariantPatched(ACCESSION, submittedVariantModified); } @@ -352,25 +365,26 @@ private void assertVariantPatched(long accession, ISubmittedVariant modifiedVari assertEquals(modifiedVariant, variantFromDb); } - @UsingDataSet(locations = {"/test-data/dbsnpSubmittedVariantEntity.json"}) @Test public void patchDbsnpSubmittedVariant() throws AccessionDeprecatedException, AccessionDoesNotExistException, AccessionMergedException, HashAlreadyExistsException { + new MongoTestDataLoader(mongoTemplate, resourceLoader).load("/test-data/dbsnpSubmittedVariantEntity.json"); + service.patch(ACCESSION_DBSNP_1, submittedVariantModified); assertVariantPatched(ACCESSION_DBSNP_1, submittedVariantModified); } - @UsingDataSet(locations = {"/test-data/submittedVariantEntity.json"}) @Test public void deprecateSubmittedVariant() throws AccessionMergedException, AccessionDoesNotExistException, AccessionDeprecatedException { + new MongoTestDataLoader(mongoTemplate, resourceLoader).load("/test-data/submittedVariantEntity.json"); + assertNotNull(service.getByAccession(ACCESSION_TO_DEPRECATE)); service.deprecate(ACCESSION_TO_DEPRECATE, DEPRECATE_REASON); assertVariantDeprecated(ACCESSION_TO_DEPRECATE, DEPRECATE_REASON); } - private void assertVariantDeprecated(long accession, String reason) - throws AccessionMergedException, AccessionDoesNotExistException, AccessionDeprecatedException { + private void assertVariantDeprecated(long accession, String reason) { IEvent lastOperation; if (accession >= accessioningMonotonicInitSs) { lastOperation = inactiveService.getLastEvent(accession); @@ -382,23 +396,24 @@ private void assertVariantDeprecated(long accession, String reason) assertNull(lastOperation.getMergedInto()); assertEquals(reason, lastOperation.getReason()); - expectedException.expect(AccessionDeprecatedException.class); - service.getByAccession(accession); + assertThrows(AccessionDeprecatedException.class, () -> service.getByAccession(accession)); } - @UsingDataSet(locations = {"/test-data/dbsnpSubmittedVariantEntity.json"}) @Test public void deprecateDnsnpSubmittedVariant() throws AccessionMergedException, AccessionDoesNotExistException, AccessionDeprecatedException { + new MongoTestDataLoader(mongoTemplate, resourceLoader).load("/test-data/dbsnpSubmittedVariantEntity.json"); + assertNotNull(service.getByAccession(ACCESSION_DBSNP_1)); service.deprecate(ACCESSION_DBSNP_1, DEPRECATE_REASON); assertVariantDeprecated(ACCESSION_DBSNP_1, DEPRECATE_REASON); } - @UsingDataSet(locations = {"/test-data/submittedVariantEntity.json"}) @Test public void mergeSubmittedVariant() throws AccessionMergedException, AccessionDoesNotExistException, AccessionDeprecatedException { + new MongoTestDataLoader(mongoTemplate, resourceLoader).load("/test-data/submittedVariantEntity.json"); + assertNotNull(service.getByAccession(ACCESSION_TO_MERGE_1)); assertNotNull(service.getByAccession(ACCESSION_TO_MERGE_2)); service.merge(ACCESSION_TO_MERGE_1, ACCESSION_TO_MERGE_2, MERGE_REASON); @@ -420,49 +435,47 @@ private void assertVariantMerged(long accessionOrigin, long accessionDestination assertNotNull(service.getByAccession(accessionDestination)); - expectedException.expect(AccessionMergedException.class); - service.getByAccession(accessionOrigin); + assertThrows(AccessionMergedException.class, () -> service.getByAccession(accessionOrigin)); } - @UsingDataSet(locations = {"/test-data/dbsnpSubmittedVariantEntity.json"}) @Test public void mergeDbsnpSubmittedVariant() throws AccessionMergedException, AccessionDoesNotExistException, AccessionDeprecatedException { + new MongoTestDataLoader(mongoTemplate, resourceLoader).load("/test-data/dbsnpSubmittedVariantEntity.json"); + assertNotNull(service.getByAccession(ACCESSION_DBSNP_1)); assertNotNull(service.getByAccession(ACCESSION_DBSNP_2)); service.merge(ACCESSION_DBSNP_1, ACCESSION_DBSNP_2, MERGE_REASON); assertVariantMerged(ACCESSION_DBSNP_1, ACCESSION_DBSNP_2, MERGE_REASON); } - @UsingDataSet(loadStrategy = LoadStrategyEnum.DELETE_ALL) - @Test(expected = UnsupportedOperationException.class) - public void exceptionWhenCreateAccessionForDbsnpVariant() throws AccessionCouldNotBeGeneratedException { + @Test + public void exceptionWhenCreateAccessionForDbsnpVariant() { DbsnpSubmittedVariantMonotonicAccessioningService accessioningServiceDbsnp = new DbsnpSubmittedVariantMonotonicAccessioningService( dbsnpAccessionGenerator, dbServiceDbsnp, new SubmittedVariantSummaryFunction(), new SHA1HashingFunction(), AccessionSaveMode.SAVE_ALL_THEN_RESOLVE); - accessioningServiceDbsnp.getOrCreate(Collections.singletonList(submittedVariant), TEST_APPLICATION_INSTANCE_ID); + assertThrows(UnsupportedOperationException.class, () -> accessioningServiceDbsnp.getOrCreate(Collections.singletonList(submittedVariant), TEST_APPLICATION_INSTANCE_ID)); } - @UsingDataSet(loadStrategy = LoadStrategyEnum.DELETE_ALL) @Test public void defaultValuesAreNotStored() throws AccessionCouldNotBeGeneratedException { boolean NOT_DEFAULT_SUPPORTED_BY_EVIDENCE = !DEFAULT_SUPPORTED_BY_EVIDENCE; SubmittedVariant submittedVariant = new SubmittedVariant("assembly", 1111, "project", "contig_1", 100, "ref", - "alt", CLUSTERED_VARIANT, - NOT_DEFAULT_SUPPORTED_BY_EVIDENCE, - DEFAULT_ASSEMBLY_MATCH, - DEFAULT_ALLELES_MATCH, - DEFAULT_VALIDATED, null); + "alt", CLUSTERED_VARIANT, + NOT_DEFAULT_SUPPORTED_BY_EVIDENCE, + DEFAULT_ASSEMBLY_MATCH, + DEFAULT_ALLELES_MATCH, + DEFAULT_VALIDATED, null); service.getOrCreate(Collections.singletonList(submittedVariant), TEST_APPLICATION_INSTANCE_ID); - List submittedVariantEntities = new ArrayList<>(); - MongoCursor submittedVariantEntitiesCursor = mongoDbFactory.getDb() - .getCollection("submittedVariantEntity") - .find().iterator(); + List submittedVariantEntities = new ArrayList<>(); + MongoCursor submittedVariantEntitiesCursor = mongoDbFactory.getMongoDatabase() + .getCollection("submittedVariantEntity") + .find().iterator(); submittedVariantEntitiesCursor.forEachRemaining(submittedVariantEntities::add); assertEquals(NOT_DEFAULT_SUPPORTED_BY_EVIDENCE, submittedVariantEntities.get(0).get("evidence")); @@ -471,20 +484,21 @@ public void defaultValuesAreNotStored() throws AccessionCouldNotBeGeneratedExcep assertEquals(null, submittedVariantEntities.get(0).get("validated")); } - @UsingDataSet(locations = {"/test-data/dbsnpSubmittedVariantEntity.json"}) @Test public void getSeveralVariantsWithSameAccession() { + new MongoTestDataLoader(mongoTemplate, resourceLoader).load("/test-data/dbsnpSubmittedVariantEntity.json"); + SubmittedVariant multiallelicSubmittedVariant1 = new SubmittedVariant("GCA_000009999.3", 9999, "DBSNP555", - "21", 20849999, "A", "G", 2200000002L, - DEFAULT_SUPPORTED_BY_EVIDENCE, - DEFAULT_ASSEMBLY_MATCH, - true, DEFAULT_VALIDATED, null); + "21", 20849999, "A", "G", 2200000002L, + DEFAULT_SUPPORTED_BY_EVIDENCE, + DEFAULT_ASSEMBLY_MATCH, + true, DEFAULT_VALIDATED, null); SubmittedVariant multiallelicSubmittedVariant2 = new SubmittedVariant("GCA_000009999.3", 9999, "DBSNP555", - "21", 20849999, "A", "C", 2200000003L, - DEFAULT_SUPPORTED_BY_EVIDENCE, - DEFAULT_ASSEMBLY_MATCH, - true, DEFAULT_VALIDATED, null); + "21", 20849999, "A", "C", 2200000003L, + DEFAULT_SUPPORTED_BY_EVIDENCE, + DEFAULT_ASSEMBLY_MATCH, + true, DEFAULT_VALIDATED, null); List> accessions = service.get( Arrays.asList(multiallelicSubmittedVariant1, multiallelicSubmittedVariant2)); @@ -492,7 +506,6 @@ public void getSeveralVariantsWithSameAccession() { assertEquals(1, accessions.stream().map(AccessionWrapper::getAccession).collect(Collectors.toSet()).size()); } - @UsingDataSet(loadStrategy = LoadStrategyEnum.DELETE_ALL) @Test public void createdDateIsReturnedInAccessionedVariant() throws AccessionCouldNotBeGeneratedException { LocalDateTime createdDate = LocalDateTime.of(2018, Month.SEPTEMBER, 18, 9, 0); diff --git a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/service/nonhuman/eva/ClusteredVariantMonotonicAccessioningServiceTest.java b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/service/nonhuman/eva/ClusteredVariantMonotonicAccessioningServiceTest.java index 3686a4df2..e94ad0347 100644 --- a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/service/nonhuman/eva/ClusteredVariantMonotonicAccessioningServiceTest.java +++ b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/service/nonhuman/eva/ClusteredVariantMonotonicAccessioningServiceTest.java @@ -17,42 +17,61 @@ */ package uk.ac.ebi.eva.accession.core.service.nonhuman.eva; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.autoconfigure.orm.jpa.DataJpaTest; +import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.ampt2d.commons.accession.core.exceptions.AccessionCouldNotBeGeneratedException; import uk.ac.ebi.ampt2d.commons.accession.core.models.GetOrCreateAccessionWrapper; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.ClusteredVariantAccessioningConfiguration; import uk.ac.ebi.eva.accession.core.model.ClusteredVariant; import uk.ac.ebi.eva.accession.core.model.IClusteredVariant; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.commons.core.models.VariantType; import java.util.Arrays; import java.util.List; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; -@RunWith(SpringRunner.class) +@ExtendWith(SpringExtension.class) @DataJpaTest @TestPropertySource("classpath:rs-accession-test.properties") -@ContextConfiguration(classes = {ClusteredVariantAccessioningConfiguration.class}) -public class ClusteredVariantMonotonicAccessioningServiceTest { +@ContextConfiguration(classes = {ClusteredVariantAccessioningConfiguration.class, MongoTestConfiguration.class}) +public class ClusteredVariantMonotonicAccessioningServiceTest extends MongoTestContainerHelper { @Autowired private ClusteredVariantMonotonicAccessioningService clusteredVariantMonotonicAccessioningService; + @Autowired + private MongoTemplate mongoTemplate; + + @BeforeEach + public void setUp() throws Exception { + mongoTemplate.getDb().drop(); + } + + @AfterEach + void cleanDb() { + mongoTemplate.getDb().drop(); + } + + @Test public void service() throws AccessionCouldNotBeGeneratedException { ClusteredVariant clusteredVariant1 = new ClusteredVariant("asm1", 100, "chr1", 1, VariantType.SNV, false, null); ClusteredVariant clusteredVariant2 = new ClusteredVariant("asm1", 100, "chr1", 5, VariantType.SNV, false, null); List> accessionedWrappers = clusteredVariantMonotonicAccessioningService.getOrCreate(Arrays.asList(clusteredVariant1, - clusteredVariant2), "test-application-instance-id"); + clusteredVariant2), "test-application-instance-id"); assertEquals(2, accessionedWrappers.size()); assertTrue(isAccessionInResults(accessionedWrappers, 3000000000L)); assertTrue(isAccessionInResults(accessionedWrappers, 3000000001L)); diff --git a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/summary/ClusteredVariantSummaryFunctionTest.java b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/summary/ClusteredVariantSummaryFunctionTest.java index 35ef54bc9..99eb79f38 100644 --- a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/summary/ClusteredVariantSummaryFunctionTest.java +++ b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/summary/ClusteredVariantSummaryFunctionTest.java @@ -17,8 +17,8 @@ */ package uk.ac.ebi.eva.accession.core.summary; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import uk.ac.ebi.eva.accession.core.model.ClusteredVariant; import uk.ac.ebi.eva.accession.core.model.IClusteredVariant; @@ -27,8 +27,8 @@ import java.time.LocalDateTime; import java.time.Month; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; public class ClusteredVariantSummaryFunctionTest { @@ -46,7 +46,7 @@ public class ClusteredVariantSummaryFunctionTest { private ClusteredVariantSummaryFunction summaryFunction; - @Before + @BeforeEach public void setUp() { summaryFunction = new ClusteredVariantSummaryFunction(); } diff --git a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/summary/SubmittedVariantSummaryFunctionTest.java b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/summary/SubmittedVariantSummaryFunctionTest.java index defe2cef1..238b669dd 100644 --- a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/summary/SubmittedVariantSummaryFunctionTest.java +++ b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/summary/SubmittedVariantSummaryFunctionTest.java @@ -17,8 +17,8 @@ */ package uk.ac.ebi.eva.accession.core.summary; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import uk.ac.ebi.eva.accession.core.model.ISubmittedVariant; import uk.ac.ebi.eva.accession.core.model.SubmittedVariant; @@ -26,8 +26,8 @@ import java.time.LocalDateTime; import java.time.Month; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; public class SubmittedVariantSummaryFunctionTest { @@ -57,7 +57,7 @@ public class SubmittedVariantSummaryFunctionTest { private SubmittedVariantSummaryFunction summaryFunction; - @Before + @BeforeEach public void setUp() { summaryFunction = new SubmittedVariantSummaryFunction(); } diff --git a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/test/configuration/DbsnpTestDataSource.java b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/test/configuration/DbsnpTestDataSource.java index dda176d9e..392f6c409 100644 --- a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/test/configuration/DbsnpTestDataSource.java +++ b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/test/configuration/DbsnpTestDataSource.java @@ -23,7 +23,6 @@ import org.springframework.jdbc.datasource.init.DatabasePopulator; import org.springframework.jdbc.datasource.init.DatabasePopulatorUtils; import org.springframework.jdbc.datasource.init.ResourceDatabasePopulator; - import uk.ac.ebi.eva.accession.core.configuration.DbsnpDataSource; import javax.sql.DataSource; diff --git a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/test/configuration/JPATestConfiguration.java b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/test/configuration/JPATestConfiguration.java new file mode 100644 index 000000000..66a846d06 --- /dev/null +++ b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/test/configuration/JPATestConfiguration.java @@ -0,0 +1,41 @@ +package uk.ac.ebi.eva.accession.core.test.configuration; + +import jakarta.persistence.EntityManagerFactory; +import org.springframework.context.annotation.Bean; +import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseBuilder; +import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseType; +import org.springframework.orm.jpa.JpaTransactionManager; +import org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean; +import org.springframework.orm.jpa.vendor.HibernateJpaVendorAdapter; +import org.springframework.transaction.PlatformTransactionManager; + +import javax.sql.DataSource; + +public class JPATestConfiguration { + @Bean + public DataSource dataSource() { + return new EmbeddedDatabaseBuilder() + .setType(EmbeddedDatabaseType.HSQL) + .build(); + } + + @Bean + public LocalContainerEntityManagerFactoryBean entityManagerFactory(DataSource dataSource) { + LocalContainerEntityManagerFactoryBean em = new LocalContainerEntityManagerFactoryBean(); + em.setDataSource(dataSource); + // Package where ContiguousIdBlockEntity (the JPA entity) lives + em.setPackagesToScan( + "uk.ac.ebi.ampt2d.commons.accession.persistence.jpa.monotonic.entities" + ); + HibernateJpaVendorAdapter vendorAdapter = new HibernateJpaVendorAdapter(); + vendorAdapter.setGenerateDdl(true); // auto-creates the schema in H2 + vendorAdapter.setShowSql(false); + em.setJpaVendorAdapter(vendorAdapter); + return em; + } + + @Bean + public PlatformTransactionManager transactionManager(EntityManagerFactory entityManagerFactory) { + return new JpaTransactionManager(entityManagerFactory); + } +} diff --git a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/test/configuration/human/MongoHumanTestConfiguration.java b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/test/configuration/human/MongoHumanTestConfiguration.java index 773f7eb12..ac6ddcaa7 100644 --- a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/test/configuration/human/MongoHumanTestConfiguration.java +++ b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/test/configuration/human/MongoHumanTestConfiguration.java @@ -17,22 +17,89 @@ */ package uk.ac.ebi.eva.accession.core.test.configuration.human; +import com.mongodb.ReadPreference; +import com.mongodb.WriteConcern; +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoClients; +import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.domain.EntityScan; import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Import; +import org.springframework.context.annotation.Primary; +import org.springframework.data.mongodb.MongoDatabaseFactory; +import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.data.mongodb.core.SimpleMongoClientDatabaseFactory; +import org.springframework.data.mongodb.core.WriteResultChecking; +import org.springframework.data.mongodb.core.convert.DefaultDbRefResolver; +import org.springframework.data.mongodb.core.convert.DefaultMongoTypeMapper; +import org.springframework.data.mongodb.core.convert.MappingMongoConverter; +import org.springframework.data.mongodb.core.convert.MongoCustomConversions; +import org.springframework.data.mongodb.core.mapping.MongoMappingContext; import org.springframework.data.mongodb.repository.config.EnableMongoRepositories; import uk.ac.ebi.eva.accession.core.configuration.ApplicationPropertiesConfiguration; import uk.ac.ebi.eva.accession.core.configuration.human.HumanClusteredVariantAccessioningConfiguration; import uk.ac.ebi.eva.accession.core.configuration.human.HumanMongoConfiguration; +import java.util.Collections; + @Configuration @EntityScan(basePackages = {"uk.ac.ebi.eva.accession.core.repository.human"}) @EnableMongoRepositories(basePackages = "uk.ac.ebi.eva.accession.core.repository.human", mongoTemplateRef = "humanMongoTemplate") @Import({HumanMongoConfiguration.class, HumanClusteredVariantAccessioningConfiguration.class, ApplicationPropertiesConfiguration.class - }) +}) @EnableConfigurationProperties(HumanMongoConfiguration.class) public class MongoHumanTestConfiguration { + + @Value("${spring.data.mongodb.uri}") + private String mongoUri; + + @Value("${spring.data.mongodb.database}") + private String database; + + @Value("${mongodb.read-preference:primary}") + private String readPreference; + + @Bean + @Primary + public MongoCustomConversions mongoCustomConversions() { + return new MongoCustomConversions(Collections.emptyList()); + } + + @Bean + @Primary + public MongoMappingContext mongoMappingContext(MongoCustomConversions conversions) { + MongoMappingContext context = new MongoMappingContext(); + context.setSimpleTypeHolder(conversions.getSimpleTypeHolder()); + context.setAutoIndexCreation(false); + return context; + } + + + @Bean("humanMongoClient") + public MongoClient humanMongoClient() { + return MongoClients.create(mongoUri); + } + + @Bean("humanFactory") + public MongoDatabaseFactory humanMongoDbFactory() { + return new SimpleMongoClientDatabaseFactory(humanMongoClient(), database); + } + + @Bean("humanMongoTemplate") + public MongoTemplate humanMongoTemplate() { + MongoMappingContext context = new MongoMappingContext(); + MappingMongoConverter converter = new MappingMongoConverter( + new DefaultDbRefResolver(humanMongoDbFactory()), context); + converter.setTypeMapper(new DefaultMongoTypeMapper(null)); + converter.afterPropertiesSet(); + MongoTemplate template = new MongoTemplate(humanMongoDbFactory(), converter); + template.setReadPreference(ReadPreference.valueOf(readPreference)); + template.setWriteConcern(WriteConcern.MAJORITY); + template.setWriteResultChecking(WriteResultChecking.EXCEPTION); + return template; + } } diff --git a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/test/configuration/nonhuman/MongoTestConfiguration.java b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/test/configuration/nonhuman/MongoTestConfiguration.java index 4d883e37f..1a357586c 100644 --- a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/test/configuration/nonhuman/MongoTestConfiguration.java +++ b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/test/configuration/nonhuman/MongoTestConfiguration.java @@ -17,12 +17,30 @@ */ package uk.ac.ebi.eva.accession.core.test.configuration.nonhuman; +import com.mongodb.ReadPreference; +import com.mongodb.WriteConcern; +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoClients; +import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.domain.EntityScan; import org.springframework.boot.test.autoconfigure.data.mongo.AutoConfigureDataMongo; +import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Primary; +import org.springframework.data.mongodb.MongoDatabaseFactory; import org.springframework.data.mongodb.config.EnableMongoAuditing; +import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.data.mongodb.core.SimpleMongoClientDatabaseFactory; +import org.springframework.data.mongodb.core.WriteResultChecking; +import org.springframework.data.mongodb.core.convert.DefaultDbRefResolver; +import org.springframework.data.mongodb.core.convert.DefaultMongoTypeMapper; +import org.springframework.data.mongodb.core.convert.MappingMongoConverter; +import org.springframework.data.mongodb.core.convert.MongoCustomConversions; +import org.springframework.data.mongodb.core.mapping.MongoMappingContext; import org.springframework.data.mongodb.repository.config.EnableMongoRepositories; +import java.util.Collections; + @Configuration @EntityScan(basePackages = {"uk.ac.ebi.eva.accession.core.persistence"}) @EnableMongoRepositories(basePackages = { @@ -32,4 +50,56 @@ @AutoConfigureDataMongo public class MongoTestConfiguration { + @Value("${spring.data.mongodb.uri}") + private String mongoUri; + + @Value("${spring.data.mongodb.database}") + private String database; + + @Value("${mongodb.read-preference:primary}") + private String readPreference; + + @Bean + @Primary + public MongoClient mongoClient() { + return MongoClients.create(mongoUri); + } + + @Bean("primaryFactory") + @Primary + public MongoDatabaseFactory mongoDbFactory() { + return new SimpleMongoClientDatabaseFactory(mongoClient(), database); + } + + @Bean + @Primary + public MongoCustomConversions mongoCustomConversions() { + return new MongoCustomConversions(Collections.emptyList()); + } + + @Bean + @Primary + public MongoMappingContext mongoMappingContext(MongoCustomConversions conversions) { + MongoMappingContext context = new MongoMappingContext(); + context.setSimpleTypeHolder(conversions.getSimpleTypeHolder()); + context.setAutoIndexCreation(false); + return context; + } + + @Bean + @Primary + public MongoTemplate mongoTemplate(MongoMappingContext mongoMappingContext, + MongoCustomConversions conversions) { + MappingMongoConverter converter = new MappingMongoConverter( + new DefaultDbRefResolver(mongoDbFactory()), mongoMappingContext); + converter.setCustomConversions(conversions); + converter.setTypeMapper(new DefaultMongoTypeMapper(null)); + converter.afterPropertiesSet(); + MongoTemplate mongoTemplate = new MongoTemplate(mongoDbFactory(), converter); + mongoTemplate.setReadPreference(ReadPreference.valueOf(readPreference)); + mongoTemplate.setWriteConcern(WriteConcern.MAJORITY); + mongoTemplate.setWriteResultChecking(WriteResultChecking.EXCEPTION); + return mongoTemplate; + } + } diff --git a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/test/rule/FixSpringMongoDbRule.java b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/test/rule/FixSpringMongoDbRule.java deleted file mode 100644 index c45eff3d4..000000000 --- a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/test/rule/FixSpringMongoDbRule.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * - * Copyright 2018 EMBL - European Bioinformatics Institute - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -package uk.ac.ebi.eva.accession.core.test.rule; - -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfiguration; -import com.lordofthejars.nosqlunit.mongodb.SpringMongoDbRule; - -/** - * Temporary fix until nosql unit rc-6 or final is released - */ -public class FixSpringMongoDbRule extends SpringMongoDbRule { - - public FixSpringMongoDbRule(MongoDbConfiguration mongoDbConfiguration) { - super(mongoDbConfiguration); - } - - public FixSpringMongoDbRule(MongoDbConfiguration mongoDbConfiguration, Object object) { - super(mongoDbConfiguration, object); - } - - @Override - public void close() { - // DO NOT CLOSE the connection (Spring will do it when destroying the context) - } - -} diff --git a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/utils/MongoTestContainerHelper.java b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/utils/MongoTestContainerHelper.java new file mode 100644 index 000000000..c3bc47b4f --- /dev/null +++ b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/utils/MongoTestContainerHelper.java @@ -0,0 +1,24 @@ +package uk.ac.ebi.eva.accession.core.utils; + +import org.springframework.boot.testcontainers.service.connection.ServiceConnection; +import org.springframework.test.context.DynamicPropertyRegistry; +import org.springframework.test.context.DynamicPropertySource; +import org.testcontainers.containers.MongoDBContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +@Testcontainers +public abstract class MongoTestContainerHelper { + + private static final String MONGO_IMAGE = "mongo:6.0"; + + @Container + @ServiceConnection + public static MongoDBContainer mongo = new MongoDBContainer(MONGO_IMAGE); + + @DynamicPropertySource + static void mongoProperties(DynamicPropertyRegistry registry) { + registry.add("spring.data.mongodb.uri", mongo::getConnectionString); + registry.add("human.mongodb.uri", mongo::getConnectionString); + } +} \ No newline at end of file diff --git a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/utils/MongoTestDataLoader.java b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/utils/MongoTestDataLoader.java new file mode 100644 index 000000000..444cda41b --- /dev/null +++ b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/utils/MongoTestDataLoader.java @@ -0,0 +1,106 @@ +package uk.ac.ebi.eva.accession.core.utils; + +import com.mongodb.client.MongoCollection; +import org.bson.BsonDocument; +import org.bson.BsonValue; +import org.springframework.core.io.Resource; +import org.springframework.core.io.ResourceLoader; +import org.springframework.data.mongodb.core.MongoTemplate; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class MongoTestDataLoader { + private final MongoTemplate mongoTemplate; + private final ResourceLoader resourceLoader; + + public MongoTestDataLoader(MongoTemplate mongoTemplate, ResourceLoader resourceLoader) { + this.mongoTemplate = mongoTemplate; + this.resourceLoader = resourceLoader; + } + + public void load(String resourcePath) { + load(resourcePath, null); + } + + public void load(String resourcePath, String collectionName) { + load(resourcePath, collectionName, false); + } + + public void load(String resourcePath, String collectionName, boolean dropCollection) { + try { + BsonDocument bsonRoot = readBsonRoot(resourcePath); + + String resolvedCollectionName; + BsonValue dataValue; + + if (collectionName != null && bsonRoot.containsKey(collectionName)) { + resolvedCollectionName = collectionName; + dataValue = bsonRoot.get(collectionName); + } else if (bsonRoot.size() == 1) { + Map.Entry entry = bsonRoot.entrySet().iterator().next(); + resolvedCollectionName = entry.getKey(); + dataValue = entry.getValue(); + } else { + throw new IllegalArgumentException("JSON file has multiple collections but no collectionName was specified. " + + "Use loadAll() to load all collections, or pass a collectionName. File: " + resourcePath); + } + + insertIntoCollection(resolvedCollectionName, dataValue, dropCollection); + } catch (IllegalArgumentException e) { + throw e; + } catch (Exception e) { + throw new RuntimeException("Failed to load test data from: " + resourcePath, e); + } + } + + public void loadAll(String resourcePath) { + loadAll(resourcePath, false); + } + + public void loadAll(String resourcePath, boolean dropCollection) { + try { + BsonDocument bsonRoot = readBsonRoot(resourcePath); + + for (Map.Entry entry : bsonRoot.entrySet()) { + insertIntoCollection(entry.getKey(), entry.getValue(), dropCollection); + } + + } catch (Exception e) { + throw new RuntimeException("Failed to load test data from: " + resourcePath, e); + } + } + private BsonDocument readBsonRoot(String resourcePath) throws Exception { + Resource resource = resourceLoader.getResource("classpath:" + resourcePath); + String content = new String(resource.getInputStream().readAllBytes(), StandardCharsets.UTF_8); + + // Normalize timezone format: +0000 -> +00:00 for BSON date parsing + content = content.replaceAll("(\\+\\d{2})(\\d{2})\"\\)", "$1:$2\")"); + + return BsonDocument.parse(content); + } + + private void insertIntoCollection(String collectionName, BsonValue dataValue, boolean dropCollection) { + List bsonDocs = new ArrayList<>(); + if (dataValue.isArray()) { + for (BsonValue item : dataValue.asArray()) { + if (item.isDocument()) { + bsonDocs.add(item.asDocument()); + } + } + } else if (dataValue.isDocument()) { + bsonDocs.add(dataValue.asDocument()); + } + + MongoCollection collection = mongoTemplate.getDb().getCollection(collectionName, BsonDocument.class); + + if (dropCollection) { + collection.deleteMany(new BsonDocument()); + } + if (!bsonDocs.isEmpty()) { + collection.insertMany(bsonDocs); + } + } +} \ No newline at end of file diff --git a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/utils/PipelineTemporaryFolderUtil.java b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/utils/PipelineTemporaryFolderUtil.java new file mode 100644 index 000000000..b1e2014bd --- /dev/null +++ b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/utils/PipelineTemporaryFolderUtil.java @@ -0,0 +1,107 @@ +package uk.ac.ebi.eva.accession.core.utils; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.io.UncheckedIOException; +import java.io.Writer; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Comparator; +import java.util.zip.GZIPOutputStream; + +public class PipelineTemporaryFolderUtil { + + private static final Logger logger = LoggerFactory.getLogger(PipelineTemporaryFolderUtil.class); + + private final Path root; + + public PipelineTemporaryFolderUtil() { + try { + this.root = Files.createTempDirectory("pipeline-test-"); + this.root.toFile().deleteOnExit(); + logger.debug("temporary folder path '{}'", root.toAbsolutePath()); + } catch (IOException e) { + throw new UncheckedIOException("Failed to create temporary folder", e); + } + } + + public File getRoot() { + return root.toFile(); + } + + public File newFolder() throws IOException { + File folder = Files.createTempDirectory(root, "tmp-dir-").toFile(); + folder.deleteOnExit(); + return folder; + } + + public File newFolder(String name) throws IOException { + File folder = root.resolve(name).toFile(); + if (!folder.exists() && !folder.mkdirs()) { + throw new IOException("Could not create directory: " + folder); + } + folder.deleteOnExit(); + return folder; + } + + public File newFile() throws IOException { + File tempFile = Files.createTempFile(root, "tmp-", null).toFile(); + tempFile.deleteOnExit(); + return tempFile; + } + + public File newFile(String name) throws IOException { + File tempFile = root.resolve(name).toFile(); + tempFile.getParentFile().mkdirs(); + if (!tempFile.exists() && !tempFile.createNewFile()) { + throw new IOException("Could not create file: " + tempFile); + } + tempFile.deleteOnExit(); + return tempFile; + } + + public File newGzipFile(String content) throws IOException { + return newGzipFile(content, null); + } + + public File newGzipFile(String content, String name) throws IOException { + File tempFile = (name == null || name.isBlank()) + ? Files.createTempFile(root, "tmp-", ".gz").toFile() + : root.resolve(name).toFile(); + + tempFile.deleteOnExit(); + + try (FileOutputStream output = new FileOutputStream(tempFile); + Writer writer = new OutputStreamWriter(new GZIPOutputStream(output), StandardCharsets.UTF_8)) { + writer.write(content); + } + + return tempFile; + } + + public void clear() { + deleteRecursively(); + } + + public void deleteRecursively() { + try { + Files.walk(root) + .sorted(Comparator.reverseOrder()) + .forEach(path -> { + try { + Files.deleteIfExists(path); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + }); + } catch (IOException e) { + throw new UncheckedIOException("Failed to delete temporary folder", e); + } + } +} \ No newline at end of file diff --git a/eva-accession-core/src/test/resources/properties/rs-accession-test.properties b/eva-accession-core/src/test/resources/properties/rs-accession-test.properties index 47f244c12..abbb7c21e 100644 --- a/eva-accession-core/src/test/resources/properties/rs-accession-test.properties +++ b/eva-accession-core/src/test/resources/properties/rs-accession-test.properties @@ -5,9 +5,6 @@ accessioning.monotonic.test-rs.blockStartValue=3000000000 accessioning.monotonic.test-rs.nextBlockInterval=1000000000 spring.data.mongodb.database=clustered-variants-test -spring.data.mongodb.host=|eva.mongo.host.test| -spring.data.mongodb.password= -spring.data.mongodb.port=27017 mongodb.read-preference=primary # See https://github.com/spring-projects/spring-boot/wiki/Spring-Boot-2.1-Release-Notes#bean-overriding diff --git a/eva-accession-core/src/test/resources/properties/ss-accession-test.properties b/eva-accession-core/src/test/resources/properties/ss-accession-test.properties index 0581ac1e4..b4151457a 100644 --- a/eva-accession-core/src/test/resources/properties/ss-accession-test.properties +++ b/eva-accession-core/src/test/resources/properties/ss-accession-test.properties @@ -13,15 +13,8 @@ spring.data.mongodb.database=submitted-variants-test # this will be replaced with a default value "localhost" when run locally # (due to the filtering attribute specified in this project's POM) # (OR) this variable can be overridden in Gitlab with "mongo" (Gitlab services are exposed under their own hostname) -spring.data.mongodb.host=|eva.mongo.host.test| -spring.data.mongodb.password= -spring.data.mongodb.port=27017 # See https://github.com/spring-projects/spring-boot/wiki/Spring-Boot-2.1-Release-Notes#bean-overriding spring.main.allow-bean-definition-overriding=true human.mongodb.database=human-variants-test -human.mongodb.host=|eva.mongo.host.test| -human.mongodb.password= -human.mongodb.port=27017 - mongodb.read-preference=primary diff --git a/eva-accession-core/src/test/resources/properties/ss-deprecation-test.properties b/eva-accession-core/src/test/resources/properties/ss-deprecation-test.properties index 22b096278..50d8e3cd5 100644 --- a/eva-accession-core/src/test/resources/properties/ss-deprecation-test.properties +++ b/eva-accession-core/src/test/resources/properties/ss-deprecation-test.properties @@ -14,15 +14,8 @@ spring.data.mongodb.database=sve-deprecation-test # this will be replaced with a default value "localhost" when run locally # (due to the filtering attribute specified in this project's POM) # (OR) this variable can be overridden in Gitlab with "mongo" (Gitlab services are exposed under their own hostname) -spring.data.mongodb.host=|eva.mongo.host.test| -spring.data.mongodb.password= -spring.data.mongodb.port=27017 # See https://github.com/spring-projects/spring-boot/wiki/Spring-Boot-2.1-Release-Notes#bean-overriding spring.main.allow-bean-definition-overriding=true human.mongodb.database=human-variants-test -human.mongodb.host=|eva.mongo.host.test| -human.mongodb.password= -human.mongodb.port=27017 - mongodb.read-preference=primary diff --git a/eva-accession-core/src/test/resources/properties/test-model.properties b/eva-accession-core/src/test/resources/properties/test-model.properties index ed3044e11..224b21c49 100644 --- a/eva-accession-core/src/test/resources/properties/test-model.properties +++ b/eva-accession-core/src/test/resources/properties/test-model.properties @@ -1,5 +1,2 @@ spring.data.mongodb.database=test-db -spring.data.mongodb.host=|eva.mongo.host.test| -spring.data.mongodb.password= -spring.data.mongodb.port=27017 mongodb.read-preference=primary \ No newline at end of file diff --git a/eva-accession-core/src/test/resources/properties/test-variants-writer.properties b/eva-accession-core/src/test/resources/properties/test-variants-writer.properties index 1d86196e5..ebf63e888 100644 --- a/eva-accession-core/src/test/resources/properties/test-variants-writer.properties +++ b/eva-accession-core/src/test/resources/properties/test-variants-writer.properties @@ -6,10 +6,6 @@ accessioning.monotonic.test-ss.blockStartValue=5000000000 accessioning.monotonic.test-ss.nextBlockInterval=1000000000 spring.data.mongodb.database=submitted-variants-test -spring.data.mongodb.host=|eva.mongo.host.test| -spring.data.mongodb.password= -spring.data.mongodb.port=27017 - mongodb.read-preference=primary # See https://github.com/spring-projects/spring-boot/wiki/Spring-Boot-2.1-Release-Notes#bean-overriding diff --git a/eva-accession-deprecate/pom.xml b/eva-accession-deprecate/pom.xml index b711b0eb4..71cf08411 100644 --- a/eva-accession-deprecate/pom.xml +++ b/eva-accession-deprecate/pom.xml @@ -2,12 +2,13 @@ + 4.0.0 + eva-accession uk.ac.ebi.eva - 0.6.57-SNAPSHOT + ${revision} - 4.0.0 eva-accession-deprecate jar @@ -28,8 +29,26 @@ - com.lordofthejars - nosqlunit-mongodb + uk.ac.ebi.eva + eva-accession-core + ${revision} + test-jar + test + + + + org.springframework.boot + spring-boot-testcontainers + test + + + org.testcontainers + mongodb + test + + + org.testcontainers + junit-jupiter test @@ -74,7 +93,4 @@ - - localhost - \ No newline at end of file diff --git a/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/batch/io/StudySubmittedVariantsReaderConfiguration.java b/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/batch/io/StudySubmittedVariantsReaderConfiguration.java index 495279961..ee471da1a 100644 --- a/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/batch/io/StudySubmittedVariantsReaderConfiguration.java +++ b/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/batch/io/StudySubmittedVariantsReaderConfiguration.java @@ -20,10 +20,9 @@ import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Import; import org.springframework.data.mongodb.core.MongoTemplate; - import uk.ac.ebi.eva.accession.core.configuration.nonhuman.MongoConfiguration; -import uk.ac.ebi.eva.accession.deprecate.configuration.BeanNames; import uk.ac.ebi.eva.accession.deprecate.batch.io.StudySubmittedVariantsReader; +import uk.ac.ebi.eva.accession.deprecate.configuration.BeanNames; import uk.ac.ebi.eva.accession.deprecate.parameters.InputParameters; @Configuration diff --git a/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/batch/jobs/DeprecateStudySubmittedVariantsJobConfiguration.java b/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/batch/jobs/DeprecateStudySubmittedVariantsJobConfiguration.java index 40d57d6d2..ebe7fa92b 100644 --- a/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/batch/jobs/DeprecateStudySubmittedVariantsJobConfiguration.java +++ b/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/batch/jobs/DeprecateStudySubmittedVariantsJobConfiguration.java @@ -17,20 +17,15 @@ import org.springframework.batch.core.Job; import org.springframework.batch.core.Step; -import org.springframework.batch.core.configuration.annotation.BatchConfigurer; import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; -import org.springframework.batch.core.configuration.annotation.JobBuilderFactory; +import org.springframework.batch.core.job.builder.JobBuilder; import org.springframework.batch.core.launch.support.RunIdIncrementer; +import org.springframework.batch.core.repository.JobRepository; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; - import uk.ac.ebi.eva.accession.deprecate.configuration.BeanNames; -import uk.ac.ebi.eva.commons.batch.configuration.SpringBoot1CompatibilityConfiguration; - -import javax.persistence.EntityManagerFactory; -import javax.sql.DataSource; @Configuration @EnableBatchProcessing @@ -41,17 +36,10 @@ public class DeprecateStudySubmittedVariantsJobConfiguration { private Step deprecateStudySubmittedVariantsStep; @Bean(BeanNames.DEPRECATE_STUDY_SUBMITTED_VARIANTS_JOB) - public Job accessionReleaseJob(JobBuilderFactory jobBuilderFactory) { - return jobBuilderFactory.get(BeanNames.DEPRECATE_STUDY_SUBMITTED_VARIANTS_JOB) - .incrementer(new RunIdIncrementer()) - .start(deprecateStudySubmittedVariantsStep) - .build(); - } - - @Bean - public BatchConfigurer configurer(DataSource dataSource, EntityManagerFactory entityManagerFactory) - throws Exception { - return SpringBoot1CompatibilityConfiguration.getSpringBoot1CompatibleBatchConfigurer(dataSource, - entityManagerFactory); + public Job accessionReleaseJob(JobRepository jobRepository) { + return new JobBuilder(BeanNames.DEPRECATE_STUDY_SUBMITTED_VARIANTS_JOB, jobRepository) + .incrementer(new RunIdIncrementer()) + .start(deprecateStudySubmittedVariantsStep) + .build(); } } diff --git a/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/batch/jobs/DeprecateSubmittedVariantsFromFileJobConfiguration.java b/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/batch/jobs/DeprecateSubmittedVariantsFromFileJobConfiguration.java index 95d9cb747..05958a588 100644 --- a/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/batch/jobs/DeprecateSubmittedVariantsFromFileJobConfiguration.java +++ b/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/batch/jobs/DeprecateSubmittedVariantsFromFileJobConfiguration.java @@ -17,19 +17,15 @@ import org.springframework.batch.core.Job; import org.springframework.batch.core.Step; -import org.springframework.batch.core.configuration.annotation.BatchConfigurer; import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; -import org.springframework.batch.core.configuration.annotation.JobBuilderFactory; +import org.springframework.batch.core.job.builder.JobBuilder; import org.springframework.batch.core.launch.support.RunIdIncrementer; +import org.springframework.batch.core.repository.JobRepository; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import uk.ac.ebi.eva.accession.deprecate.configuration.BeanNames; -import uk.ac.ebi.eva.commons.batch.configuration.SpringBoot1CompatibilityConfiguration; - -import javax.persistence.EntityManagerFactory; -import javax.sql.DataSource; @Configuration @EnableBatchProcessing @@ -40,17 +36,10 @@ public class DeprecateSubmittedVariantsFromFileJobConfiguration { private Step deprecateSubmittedVariantsFromFileStep; @Bean(BeanNames.DEPRECATE_SUBMITTED_VARIANTS_FROM_FILE_JOB) - public Job deprecateStudySubmittedVariantsFromFileJob(JobBuilderFactory jobBuilderFactory) { - return jobBuilderFactory.get(BeanNames.DEPRECATE_SUBMITTED_VARIANTS_FROM_FILE_JOB) + public Job deprecateStudySubmittedVariantsFromFileJob(JobRepository jobRepository) { + return new JobBuilder(BeanNames.DEPRECATE_SUBMITTED_VARIANTS_FROM_FILE_JOB, jobRepository) .incrementer(new RunIdIncrementer()) .start(deprecateSubmittedVariantsFromFileStep) .build(); } - - @Bean - public BatchConfigurer configurer(DataSource dataSource, EntityManagerFactory entityManagerFactory) - throws Exception { - return SpringBoot1CompatibilityConfiguration.getSpringBoot1CompatibleBatchConfigurer(dataSource, - entityManagerFactory); - } } diff --git a/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/batch/listeners/ListenerConfiguration.java b/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/batch/listeners/ListenerConfiguration.java index b0b3774f6..807e8a77e 100644 --- a/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/batch/listeners/ListenerConfiguration.java +++ b/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/batch/listeners/ListenerConfiguration.java @@ -21,14 +21,13 @@ import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Import; import org.springframework.web.client.RestTemplate; - import uk.ac.ebi.eva.accession.clustering.metric.ClusteringMetricCompute; -import uk.ac.ebi.eva.accession.deprecate.parameters.InputParameters; import uk.ac.ebi.eva.accession.core.batch.io.SubmittedVariantDeprecationWriter; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; -import uk.ac.ebi.eva.accession.deprecate.configuration.BeanNames; import uk.ac.ebi.eva.accession.deprecate.batch.listeners.DeprecationStepProgressListener; +import uk.ac.ebi.eva.accession.deprecate.configuration.BeanNames; import uk.ac.ebi.eva.accession.deprecate.configuration.InputParametersConfiguration; +import uk.ac.ebi.eva.accession.deprecate.parameters.InputParameters; import uk.ac.ebi.eva.metrics.configuration.MetricConfiguration; import uk.ac.ebi.eva.metrics.count.CountServiceParameters; import uk.ac.ebi.eva.metrics.metric.MetricCompute; diff --git a/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/batch/steps/DeprecateStudySubmittedVariantsStepConfiguration.java b/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/batch/steps/DeprecateStudySubmittedVariantsStepConfiguration.java index 86eaed6d8..37261c120 100644 --- a/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/batch/steps/DeprecateStudySubmittedVariantsStepConfiguration.java +++ b/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/batch/steps/DeprecateStudySubmittedVariantsStepConfiguration.java @@ -18,7 +18,8 @@ import org.springframework.batch.core.Step; import org.springframework.batch.core.StepExecutionListener; import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; -import org.springframework.batch.core.configuration.annotation.StepBuilderFactory; +import org.springframework.batch.core.repository.JobRepository; +import org.springframework.batch.core.step.builder.StepBuilder; import org.springframework.batch.core.step.tasklet.TaskletStep; import org.springframework.batch.item.ItemStreamReader; import org.springframework.batch.item.ItemWriter; @@ -27,7 +28,7 @@ import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; - +import org.springframework.transaction.PlatformTransactionManager; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; import uk.ac.ebi.eva.accession.deprecate.configuration.BeanNames; @@ -48,10 +49,11 @@ public class DeprecateStudySubmittedVariantsStepConfiguration { private StepExecutionListener progressListener; @Bean(BeanNames.DEPRECATE_STUDY_SUBMITTED_VARIANTS_STEP) - public Step deprecateClusteredVariantsStep(StepBuilderFactory stepBuilderFactory, + public Step deprecateClusteredVariantsStep(JobRepository jobRepository, + PlatformTransactionManager transactionManager, SimpleCompletionPolicy chunkSizeCompletionPolicy) { - TaskletStep step = stepBuilderFactory.get(BeanNames.DEPRECATE_STUDY_SUBMITTED_VARIANTS_STEP) - .chunk(chunkSizeCompletionPolicy) + TaskletStep step = new StepBuilder(BeanNames.DEPRECATE_STUDY_SUBMITTED_VARIANTS_STEP, jobRepository) + .chunk(chunkSizeCompletionPolicy, transactionManager) .reader(studySubmittedVariantsReader) .writer(submittedVariantDeprecationWriter) .listener(progressListener) diff --git a/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/batch/steps/DeprecateSubmittedVariantsFromFileStepConfiguration.java b/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/batch/steps/DeprecateSubmittedVariantsFromFileStepConfiguration.java index 1ef5835fa..aad2dfedd 100644 --- a/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/batch/steps/DeprecateSubmittedVariantsFromFileStepConfiguration.java +++ b/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/batch/steps/DeprecateSubmittedVariantsFromFileStepConfiguration.java @@ -18,7 +18,8 @@ import org.springframework.batch.core.Step; import org.springframework.batch.core.StepExecutionListener; import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; -import org.springframework.batch.core.configuration.annotation.StepBuilderFactory; +import org.springframework.batch.core.repository.JobRepository; +import org.springframework.batch.core.step.builder.StepBuilder; import org.springframework.batch.core.step.tasklet.TaskletStep; import org.springframework.batch.item.ItemStreamReader; import org.springframework.batch.item.ItemWriter; @@ -27,6 +28,7 @@ import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.transaction.PlatformTransactionManager; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; import uk.ac.ebi.eva.accession.deprecate.configuration.BeanNames; @@ -47,10 +49,11 @@ public class DeprecateSubmittedVariantsFromFileStepConfiguration { private StepExecutionListener progressListener; @Bean(BeanNames.DEPRECATE_SUBMITTED_VARIANTS_FROM_FILE_STEP) - public Step deprecateSubmittedVariantsFromFileStep(StepBuilderFactory stepBuilderFactory, - SimpleCompletionPolicy chunkSizeCompletionPolicy) { - TaskletStep step = stepBuilderFactory.get(BeanNames.DEPRECATE_SUBMITTED_VARIANTS_FROM_FILE_STEP) - .chunk(chunkSizeCompletionPolicy) + public Step deprecateSubmittedVariantsFromFileStep(JobRepository jobRepository, + PlatformTransactionManager transactionManager, + SimpleCompletionPolicy chunkSizeCompletionPolicy) { + TaskletStep step = new StepBuilder(BeanNames.DEPRECATE_SUBMITTED_VARIANTS_FROM_FILE_STEP, jobRepository) + .chunk(chunkSizeCompletionPolicy, transactionManager) .reader(submittedVariantsFileReader) .writer(submittedVariantDeprecationWriter) .listener(progressListener) diff --git a/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/MongoTestDatabaseSetup.java b/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/MongoTestDatabaseSetup.java index 56efe9658..3488818f1 100644 --- a/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/MongoTestDatabaseSetup.java +++ b/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/MongoTestDatabaseSetup.java @@ -32,8 +32,8 @@ import java.util.Arrays; import java.util.function.Function; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; public class MongoTestDatabaseSetup { diff --git a/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/batch/io/StudySubmittedVariantsReaderTest.java b/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/batch/io/StudySubmittedVariantsReaderTest.java index 29eb3e093..ea6b98198 100644 --- a/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/batch/io/StudySubmittedVariantsReaderTest.java +++ b/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/batch/io/StudySubmittedVariantsReaderTest.java @@ -15,43 +15,34 @@ */ package uk.ac.ebi.eva.accession.deprecate.batch.io; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; -import com.mongodb.MongoClient; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.batch.item.ExecutionContext; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.ApplicationContext; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; - +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.MongoConfiguration; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.deprecate.test.configuration.MongoTestConfiguration; -import uk.ac.ebi.eva.accession.deprecate.test.rule.FixSpringMongoDbRule; import java.util.ArrayList; import java.util.List; import static junit.framework.TestCase.assertTrue; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import static uk.ac.ebi.eva.accession.deprecate.MongoTestDatabaseSetup.ASSEMBLY; -import static uk.ac.ebi.eva.accession.deprecate.MongoTestDatabaseSetup.populateTestDB; import static uk.ac.ebi.eva.accession.deprecate.MongoTestDatabaseSetup.STUDY1; +import static uk.ac.ebi.eva.accession.deprecate.MongoTestDatabaseSetup.populateTestDB; -@RunWith(SpringRunner.class) +@ExtendWith(SpringExtension.class) @TestPropertySource("classpath:study-submitted-variants-test.properties") @ContextConfiguration(classes = {MongoConfiguration.class, MongoTestConfiguration.class}) -public class StudySubmittedVariantsReaderTest { - - private static final String TEST_DB = "test-db"; - +public class StudySubmittedVariantsReaderTest extends MongoTestContainerHelper { private static final String ID_1 = "hash5"; private static final String ID_2 = "hash6"; @@ -64,38 +55,28 @@ public class StudySubmittedVariantsReaderTest { private StudySubmittedVariantsReader reader; - @Autowired - private MongoClient mongoClient; - @Autowired private MongoTemplate mongoTemplate; - //Required by nosql-unit - @Autowired - private ApplicationContext applicationContext; - - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName(TEST_DB).build()); - - @Before + @BeforeEach public void setUp() { + mongoTemplate.getDb().drop(); populateTestDB(this.mongoTemplate); executionContext = new ExecutionContext(); reader = new StudySubmittedVariantsReader(ASSEMBLY, STUDY1, mongoTemplate, CHUNK_SIZE); reader.open(executionContext); } - @After + @AfterEach public void tearDown() { + mongoTemplate.getDb().drop(); reader.close(); - mongoClient.dropDatabase(TEST_DB); } @Test public void readStudySubmittedVariants() { List variants = readIntoList(); - assertEquals(3, variants.size()); + assertEquals(3, variants.size()); assertTrue(variants.stream().anyMatch(x -> x.getId().equals(ID_1))); assertTrue(variants.stream().anyMatch(x -> x.getId().equals(ID_2))); assertTrue(variants.stream().anyMatch(x -> x.getId().equals(ID_3))); diff --git a/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/configuration/jobs/DeprecateStudySubmittedVariantsJobConfigurationTest.java b/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/configuration/jobs/DeprecateStudySubmittedVariantsJobConfigurationTest.java index 01ed86b5f..2c44eb795 100644 --- a/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/configuration/jobs/DeprecateStudySubmittedVariantsJobConfigurationTest.java +++ b/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/configuration/jobs/DeprecateStudySubmittedVariantsJobConfigurationTest.java @@ -15,14 +15,10 @@ */ package uk.ac.ebi.eva.accession.deprecate.configuration.jobs; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; -import com.mongodb.MongoClient; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.batch.core.BatchStatus; import org.springframework.batch.core.JobExecution; import org.springframework.batch.core.StepExecution; @@ -32,37 +28,34 @@ import org.springframework.batch.test.JobRepositoryTestUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.context.ApplicationContext; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.eva.accession.core.runner.CommandLineRunnerUtils; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.deprecate.MongoTestDatabaseSetup; +import uk.ac.ebi.eva.accession.deprecate.test.configuration.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.accession.deprecate.test.configuration.BatchTestConfiguration; import uk.ac.ebi.eva.accession.deprecate.test.configuration.MongoTestConfiguration; -import uk.ac.ebi.eva.accession.deprecate.test.rule.FixSpringMongoDbRule; -import javax.sql.DataSource; import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.stream.Collectors; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; import static uk.ac.ebi.eva.accession.deprecate.configuration.BeanNames.DEPRECATE_STUDY_SUBMITTED_VARIANTS_JOB; import static uk.ac.ebi.eva.accession.deprecate.configuration.BeanNames.DEPRECATE_STUDY_SUBMITTED_VARIANTS_STEP; import static uk.ac.ebi.eva.accession.deprecate.test.configuration.BatchTestConfiguration.JOB_LAUNCHER_FROM_MONGO; -@RunWith(SpringRunner.class) -@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class}) +@ExtendWith(SpringExtension.class) +@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class, + BatchJobRepositoryTestConfiguration.class}) @TestPropertySource("classpath:study-submitted-variants-test.properties") -public class DeprecateStudySubmittedVariantsJobConfigurationTest { - - private static final String TEST_DB = "test-db"; - +public class DeprecateStudySubmittedVariantsJobConfigurationTest extends MongoTestContainerHelper { @Autowired @Qualifier(JOB_LAUNCHER_FROM_MONGO) private JobLauncherTestUtils jobLauncherTestUtilsFromMongo; @@ -73,36 +66,22 @@ public class DeprecateStudySubmittedVariantsJobConfigurationTest { @Autowired private JobRepository jobRepository; - @Autowired - private DataSource datasource; - - @Autowired - private MongoClient mongoClient; - @Autowired private MongoTemplate mongoTemplate; private JobRepositoryTestUtils jobRepositoryTestUtils; - //Required by nosql-unit - @Autowired - private ApplicationContext applicationContext; - - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName(TEST_DB).build()); - - @Before - public void setUp() throws Exception { - this.mongoClient.dropDatabase(TEST_DB); + @BeforeEach + public void setUp() { + mongoTemplate.getDb().drop(); MongoTestDatabaseSetup.populateTestDB(this.mongoTemplate); - jobRepositoryTestUtils = new JobRepositoryTestUtils(jobRepository, datasource); + jobRepositoryTestUtils = new JobRepositoryTestUtils(jobRepository); jobRepositoryTestUtils.removeJobExecutions(); } - @After + @AfterEach public void tearDown() { - this.mongoClient.dropDatabase(TEST_DB); + mongoTemplate.getDb().drop(); jobRepositoryTestUtils.removeJobExecutions(); } @@ -134,18 +113,18 @@ public void restartCompletedJobThatIsAlreadyInTheRepository() throws Exception { MongoTestDatabaseSetup.assertPostDeprecationDatabaseState(this.mongoTemplate); long instanceIdFirstJob = CommandLineRunnerUtils.getLastJobExecution(DEPRECATE_STUDY_SUBMITTED_VARIANTS_JOB, - jobExplorer, - jobExecution.getJobParameters()) - .getJobInstance().getInstanceId(); + jobExplorer, + jobExecution.getJobParameters()) + .getJobInstance().getInstanceId(); jobExecution = jobLauncherTestUtilsFromMongo.launchJob(); expectedSteps = Collections.singletonList(DEPRECATE_STUDY_SUBMITTED_VARIANTS_STEP); assertStepsExecuted(expectedSteps, jobExecution); assertEquals(BatchStatus.COMPLETED, jobExecution.getStatus()); long instanceIdSecondJob = CommandLineRunnerUtils.getLastJobExecution(DEPRECATE_STUDY_SUBMITTED_VARIANTS_JOB, - jobExplorer, - jobExecution.getJobParameters()) - .getJobInstance().getInstanceId(); + jobExplorer, + jobExecution.getJobParameters()) + .getJobInstance().getInstanceId(); assertNotEquals(instanceIdSecondJob, instanceIdFirstJob); MongoTestDatabaseSetup.assertPostDeprecationDatabaseState(this.mongoTemplate); } diff --git a/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/configuration/jobs/DeprecateSubmittedVariantsFromFileJobConfigurationTest.java b/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/configuration/jobs/DeprecateSubmittedVariantsFromFileJobConfigurationTest.java index 796759f8f..693ff64f1 100644 --- a/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/configuration/jobs/DeprecateSubmittedVariantsFromFileJobConfigurationTest.java +++ b/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/configuration/jobs/DeprecateSubmittedVariantsFromFileJobConfigurationTest.java @@ -15,14 +15,10 @@ */ package uk.ac.ebi.eva.accession.deprecate.configuration.jobs; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; -import com.mongodb.MongoClient; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.batch.core.BatchStatus; import org.springframework.batch.core.JobExecution; import org.springframework.batch.core.StepExecution; @@ -32,37 +28,34 @@ import org.springframework.batch.test.JobRepositoryTestUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.context.ApplicationContext; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.eva.accession.core.runner.CommandLineRunnerUtils; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.deprecate.MongoTestDatabaseSetup; +import uk.ac.ebi.eva.accession.deprecate.test.configuration.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.accession.deprecate.test.configuration.BatchTestConfiguration; import uk.ac.ebi.eva.accession.deprecate.test.configuration.MongoTestConfiguration; -import uk.ac.ebi.eva.accession.deprecate.test.rule.FixSpringMongoDbRule; -import javax.sql.DataSource; import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.stream.Collectors; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; import static uk.ac.ebi.eva.accession.deprecate.configuration.BeanNames.DEPRECATE_SUBMITTED_VARIANTS_FROM_FILE_JOB; import static uk.ac.ebi.eva.accession.deprecate.configuration.BeanNames.DEPRECATE_SUBMITTED_VARIANTS_FROM_FILE_STEP; import static uk.ac.ebi.eva.accession.deprecate.test.configuration.BatchTestConfiguration.JOB_LAUNCHER_FROM_FILE; -@RunWith(SpringRunner.class) -@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class}) +@ExtendWith(SpringExtension.class) +@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class, + BatchJobRepositoryTestConfiguration.class}) @TestPropertySource("classpath:deprecate-submitted-variants-from-file-test.properties") -public class DeprecateSubmittedVariantsFromFileJobConfigurationTest { - - private static final String TEST_DB = "test-db"; - +public class DeprecateSubmittedVariantsFromFileJobConfigurationTest extends MongoTestContainerHelper { @Autowired @Qualifier(JOB_LAUNCHER_FROM_FILE) private JobLauncherTestUtils jobLauncherTestUtilsFromFile; @@ -73,36 +66,22 @@ public class DeprecateSubmittedVariantsFromFileJobConfigurationTest { @Autowired private JobRepository jobRepository; - @Autowired - private DataSource datasource; - - @Autowired - private MongoClient mongoClient; - @Autowired private MongoTemplate mongoTemplate; private JobRepositoryTestUtils jobRepositoryTestUtils; - //Required by nosql-unit - @Autowired - private ApplicationContext applicationContext; - - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName(TEST_DB).build()); - - @Before - public void setUp() throws Exception { - this.mongoClient.dropDatabase(TEST_DB); + @BeforeEach + public void setUp() { + mongoTemplate.getDb().drop(); MongoTestDatabaseSetup.populateTestDBForFile(this.mongoTemplate); - jobRepositoryTestUtils = new JobRepositoryTestUtils(jobRepository, datasource); + jobRepositoryTestUtils = new JobRepositoryTestUtils(jobRepository); jobRepositoryTestUtils.removeJobExecutions(); } - @After + @AfterEach public void tearDown() { - this.mongoClient.dropDatabase(TEST_DB); + mongoTemplate.getDb().drop(); jobRepositoryTestUtils.removeJobExecutions(); } diff --git a/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/configuration/steps/DeprecateStudySubmittedVariantsStepConfigurationTest.java b/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/configuration/steps/DeprecateStudySubmittedVariantsStepConfigurationTest.java index 469370fc5..67ad9f95a 100644 --- a/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/configuration/steps/DeprecateStudySubmittedVariantsStepConfigurationTest.java +++ b/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/configuration/steps/DeprecateStudySubmittedVariantsStepConfigurationTest.java @@ -15,68 +15,50 @@ */ package uk.ac.ebi.eva.accession.deprecate.configuration.steps; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; -import com.mongodb.MongoClient; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.batch.core.BatchStatus; import org.springframework.batch.core.JobExecution; import org.springframework.batch.test.JobLauncherTestUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.context.ApplicationContext; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; - +import org.springframework.test.context.junit.jupiter.SpringExtension; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.deprecate.MongoTestDatabaseSetup; import uk.ac.ebi.eva.accession.deprecate.configuration.BeanNames; +import uk.ac.ebi.eva.accession.deprecate.test.configuration.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.accession.deprecate.test.configuration.BatchTestConfiguration; import uk.ac.ebi.eva.accession.deprecate.test.configuration.MongoTestConfiguration; -import uk.ac.ebi.eva.accession.deprecate.test.rule.FixSpringMongoDbRule; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import static uk.ac.ebi.eva.accession.deprecate.test.configuration.BatchTestConfiguration.JOB_LAUNCHER_FROM_MONGO; -@RunWith(SpringRunner.class) -@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class}) +@ExtendWith(SpringExtension.class) +@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class, + BatchJobRepositoryTestConfiguration.class}) @TestPropertySource("classpath:study-submitted-variants-test.properties") -public class DeprecateStudySubmittedVariantsStepConfigurationTest { - - private static final String TEST_DB = "test-db"; - +public class DeprecateStudySubmittedVariantsStepConfigurationTest extends MongoTestContainerHelper { @Autowired @Qualifier(JOB_LAUNCHER_FROM_MONGO) private JobLauncherTestUtils jobLauncherTestUtilsFromMongo; - @Autowired - private MongoClient mongoClient; - @Autowired private MongoTemplate mongoTemplate; - //Required by nosql-unit - @Autowired - private ApplicationContext applicationContext; - - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName(TEST_DB).build()); - - @Before + @BeforeEach public void setUp() { - this.mongoClient.dropDatabase(TEST_DB); + mongoTemplate.getDb().drop(); MongoTestDatabaseSetup.populateTestDB(this.mongoTemplate); } - @After + @AfterEach public void tearDown() { - this.mongoClient.dropDatabase(TEST_DB); + mongoTemplate.getDb().drop(); } @Test diff --git a/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/configuration/steps/DeprecateSubmittedVariantsFromFileStepConfigurationTest.java b/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/configuration/steps/DeprecateSubmittedVariantsFromFileStepConfigurationTest.java index 7f319abeb..05ab6f356 100644 --- a/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/configuration/steps/DeprecateSubmittedVariantsFromFileStepConfigurationTest.java +++ b/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/configuration/steps/DeprecateSubmittedVariantsFromFileStepConfigurationTest.java @@ -15,67 +15,50 @@ */ package uk.ac.ebi.eva.accession.deprecate.configuration.steps; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; -import com.mongodb.MongoClient; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.batch.core.BatchStatus; import org.springframework.batch.core.JobExecution; import org.springframework.batch.test.JobLauncherTestUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.context.ApplicationContext; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.deprecate.MongoTestDatabaseSetup; import uk.ac.ebi.eva.accession.deprecate.configuration.BeanNames; +import uk.ac.ebi.eva.accession.deprecate.test.configuration.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.accession.deprecate.test.configuration.BatchTestConfiguration; import uk.ac.ebi.eva.accession.deprecate.test.configuration.MongoTestConfiguration; -import uk.ac.ebi.eva.accession.deprecate.test.rule.FixSpringMongoDbRule; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import static uk.ac.ebi.eva.accession.deprecate.test.configuration.BatchTestConfiguration.JOB_LAUNCHER_FROM_FILE; -@RunWith(SpringRunner.class) -@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class}) +@ExtendWith(SpringExtension.class) +@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class, + BatchJobRepositoryTestConfiguration.class}) @TestPropertySource("classpath:deprecate-submitted-variants-from-file-test.properties") -public class DeprecateSubmittedVariantsFromFileStepConfigurationTest { - - private static final String TEST_DB = "test-db"; - +public class DeprecateSubmittedVariantsFromFileStepConfigurationTest extends MongoTestContainerHelper { @Autowired @Qualifier(JOB_LAUNCHER_FROM_FILE) private JobLauncherTestUtils jobLauncherTestUtilsFromFile; - @Autowired - private MongoClient mongoClient; - @Autowired private MongoTemplate mongoTemplate; - //Required by nosql-unit - @Autowired - private ApplicationContext applicationContext; - - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName(TEST_DB).build()); - - @Before + @BeforeEach public void setUp() { - this.mongoClient.dropDatabase(TEST_DB); + mongoTemplate.getDb().drop(); MongoTestDatabaseSetup.populateTestDBForFile(this.mongoTemplate); } - @After + @AfterEach public void tearDown() { - this.mongoClient.dropDatabase(TEST_DB); + mongoTemplate.getDb().drop(); } @Test diff --git a/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/test/configuration/BatchJobRepositoryTestConfiguration.java b/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/test/configuration/BatchJobRepositoryTestConfiguration.java new file mode 100644 index 000000000..854541a72 --- /dev/null +++ b/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/test/configuration/BatchJobRepositoryTestConfiguration.java @@ -0,0 +1,22 @@ +package uk.ac.ebi.eva.accession.deprecate.test.configuration; + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Primary; +import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseBuilder; +import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseType; + +import javax.sql.DataSource; + +@Configuration +public class BatchJobRepositoryTestConfiguration { + + @Bean + @Primary + public DataSource dataSource() { + return new EmbeddedDatabaseBuilder() + .setType(EmbeddedDatabaseType.HSQL) + .addScript("org/springframework/batch/core/schema-hsqldb.sql") + .build(); + } +} \ No newline at end of file diff --git a/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/test/configuration/BatchTestConfiguration.java b/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/test/configuration/BatchTestConfiguration.java index 05b7ccab3..f30f6de12 100644 --- a/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/test/configuration/BatchTestConfiguration.java +++ b/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/test/configuration/BatchTestConfiguration.java @@ -16,8 +16,9 @@ package uk.ac.ebi.eva.accession.deprecate.test.configuration; import org.springframework.batch.core.Job; +import org.springframework.batch.core.launch.JobLauncher; +import org.springframework.batch.core.repository.JobRepository; import org.springframework.batch.test.JobLauncherTestUtils; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.context.annotation.Bean; @@ -27,15 +28,15 @@ import uk.ac.ebi.eva.accession.deprecate.configuration.batch.io.StudySubmittedVariantsFileReaderConfiguration; import uk.ac.ebi.eva.accession.deprecate.configuration.batch.io.StudySubmittedVariantsReaderConfiguration; import uk.ac.ebi.eva.accession.deprecate.configuration.batch.io.SubmittedVariantDeprecationWriterConfiguration; -import uk.ac.ebi.eva.accession.deprecate.configuration.batch.jobs.DeprecateSubmittedVariantsFromFileJobConfiguration; import uk.ac.ebi.eva.accession.deprecate.configuration.batch.jobs.DeprecateStudySubmittedVariantsJobConfiguration; +import uk.ac.ebi.eva.accession.deprecate.configuration.batch.jobs.DeprecateSubmittedVariantsFromFileJobConfiguration; import uk.ac.ebi.eva.accession.deprecate.configuration.batch.listeners.ListenerConfiguration; import uk.ac.ebi.eva.accession.deprecate.configuration.batch.policies.ChunkSizeCompletionPolicyConfiguration; -import uk.ac.ebi.eva.accession.deprecate.configuration.batch.steps.DeprecateSubmittedVariantsFromFileStepConfiguration; import uk.ac.ebi.eva.accession.deprecate.configuration.batch.steps.DeprecateStudySubmittedVariantsStepConfiguration; +import uk.ac.ebi.eva.accession.deprecate.configuration.batch.steps.DeprecateSubmittedVariantsFromFileStepConfiguration; -import static uk.ac.ebi.eva.accession.deprecate.configuration.BeanNames.DEPRECATE_SUBMITTED_VARIANTS_FROM_FILE_JOB; import static uk.ac.ebi.eva.accession.deprecate.configuration.BeanNames.DEPRECATE_STUDY_SUBMITTED_VARIANTS_JOB; +import static uk.ac.ebi.eva.accession.deprecate.configuration.BeanNames.DEPRECATE_SUBMITTED_VARIANTS_FROM_FILE_JOB; @EnableAutoConfiguration @Import({MongoConfiguration.class, @@ -51,31 +52,27 @@ ListenerConfiguration.class }) public class BatchTestConfiguration { - public static final String JOB_LAUNCHER_FROM_MONGO = "JOB_LAUNCHER_FROM_MONGO"; public static final String JOB_LAUNCHER_FROM_FILE = "JOB_LAUNCHER_FROM_FILE"; @Bean(JOB_LAUNCHER_FROM_MONGO) - public JobLauncherTestUtils jobLauncherTestUtilsFromMongo() { - - return new JobLauncherTestUtils() { - @Override - @Autowired - public void setJob(@Qualifier(DEPRECATE_STUDY_SUBMITTED_VARIANTS_JOB) Job job) { - super.setJob(job); - } - }; + public JobLauncherTestUtils jobLauncherTestUtilsFromMongo(JobLauncher jobLauncher, JobRepository jobRepository, + @Qualifier(DEPRECATE_STUDY_SUBMITTED_VARIANTS_JOB) Job job) { + JobLauncherTestUtils utils = new JobLauncherTestUtils(); + utils.setJobLauncher(jobLauncher); + utils.setJobRepository(jobRepository); + utils.setJob(job); + return utils; } @Bean(JOB_LAUNCHER_FROM_FILE) - public JobLauncherTestUtils jobLauncherTestUtilsFromFile() { + public JobLauncherTestUtils jobLauncherTestUtilsFromFile(JobLauncher jobLauncher, JobRepository jobRepository, + @Qualifier(DEPRECATE_SUBMITTED_VARIANTS_FROM_FILE_JOB) Job job) { + JobLauncherTestUtils utils = new JobLauncherTestUtils(); + utils.setJobLauncher(jobLauncher); + utils.setJobRepository(jobRepository); + utils.setJob(job); + return utils; - return new JobLauncherTestUtils() { - @Override - @Autowired - public void setJob(@Qualifier(DEPRECATE_SUBMITTED_VARIANTS_FROM_FILE_JOB) Job job) { - super.setJob(job); - } - }; } } diff --git a/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/test/configuration/MongoTestConfiguration.java b/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/test/configuration/MongoTestConfiguration.java index 06aa1e647..08c2a4774 100644 --- a/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/test/configuration/MongoTestConfiguration.java +++ b/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/test/configuration/MongoTestConfiguration.java @@ -17,12 +17,30 @@ */ package uk.ac.ebi.eva.accession.deprecate.test.configuration; +import com.mongodb.ReadPreference; +import com.mongodb.WriteConcern; +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoClients; +import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.domain.EntityScan; import org.springframework.boot.test.autoconfigure.data.mongo.AutoConfigureDataMongo; +import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Primary; +import org.springframework.data.mongodb.MongoDatabaseFactory; import org.springframework.data.mongodb.config.EnableMongoAuditing; +import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.data.mongodb.core.SimpleMongoClientDatabaseFactory; +import org.springframework.data.mongodb.core.WriteResultChecking; +import org.springframework.data.mongodb.core.convert.DefaultDbRefResolver; +import org.springframework.data.mongodb.core.convert.DefaultMongoTypeMapper; +import org.springframework.data.mongodb.core.convert.MappingMongoConverter; +import org.springframework.data.mongodb.core.convert.MongoCustomConversions; +import org.springframework.data.mongodb.core.mapping.MongoMappingContext; import org.springframework.data.mongodb.repository.config.EnableMongoRepositories; +import java.util.Collections; + @Configuration @EntityScan(basePackages = {"uk.ac.ebi.eva.accession.core.persistence"}) @EnableMongoRepositories(basePackages = { @@ -32,4 +50,56 @@ @AutoConfigureDataMongo public class MongoTestConfiguration { + @Value("${spring.data.mongodb.uri}") + private String mongoUri; + + @Value("${spring.data.mongodb.database}") + private String database; + + @Value("${mongodb.read-preference:primary}") + private String readPreference; + + @Bean + @Primary + public MongoClient mongoClient() { + return MongoClients.create(mongoUri); + } + + @Bean("primaryFactory") + @Primary + public MongoDatabaseFactory mongoDbFactory() { + return new SimpleMongoClientDatabaseFactory(mongoClient(), database); + } + + @Bean + @Primary + public MongoCustomConversions mongoCustomConversions() { + return new MongoCustomConversions(Collections.emptyList()); + } + + @Bean + @Primary + public MongoMappingContext mongoMappingContext(MongoCustomConversions conversions) { + MongoMappingContext context = new MongoMappingContext(); + context.setSimpleTypeHolder(conversions.getSimpleTypeHolder()); + context.setAutoIndexCreation(false); + return context; + } + + @Bean + @Primary + public MongoTemplate mongoTemplate(MongoMappingContext mongoMappingContext, + MongoCustomConversions conversions) { + MappingMongoConverter converter = new MappingMongoConverter( + new DefaultDbRefResolver(mongoDbFactory()), mongoMappingContext); + converter.setCustomConversions(conversions); + converter.setTypeMapper(new DefaultMongoTypeMapper(null)); + converter.afterPropertiesSet(); + MongoTemplate mongoTemplate = new MongoTemplate(mongoDbFactory(), converter); + mongoTemplate.setReadPreference(ReadPreference.valueOf(readPreference)); + mongoTemplate.setWriteConcern(WriteConcern.MAJORITY); + mongoTemplate.setWriteResultChecking(WriteResultChecking.EXCEPTION); + return mongoTemplate; + } + } \ No newline at end of file diff --git a/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/test/rule/FixSpringMongoDbRule.java b/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/test/rule/FixSpringMongoDbRule.java deleted file mode 100644 index a4dffb32b..000000000 --- a/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/test/rule/FixSpringMongoDbRule.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * - * Copyright 2022 EMBL - European Bioinformatics Institute - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -package uk.ac.ebi.eva.accession.deprecate.test.rule; - -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfiguration; -import com.lordofthejars.nosqlunit.mongodb.SpringMongoDbRule; - -/** - * Temporary fix until nosql unit rc-6 or final is released - * TODO Move this to the core module (used in the release and deprecate module) - */ -public class FixSpringMongoDbRule extends SpringMongoDbRule { - - public FixSpringMongoDbRule(MongoDbConfiguration mongoDbConfiguration) { - super(mongoDbConfiguration); - } - - public FixSpringMongoDbRule(MongoDbConfiguration mongoDbConfiguration, Object object) { - super(mongoDbConfiguration, object); - } - - @Override - public void close() { - // DO NOT CLOSE the connection (Spring will do it when destroying the context) - } - -} diff --git a/eva-accession-deprecate/src/test/resources/deprecate-submitted-variants-from-file-test.properties b/eva-accession-deprecate/src/test/resources/deprecate-submitted-variants-from-file-test.properties index 68229ca2e..5ea0e7638 100644 --- a/eva-accession-deprecate/src/test/resources/deprecate-submitted-variants-from-file-test.properties +++ b/eva-accession-deprecate/src/test/resources/deprecate-submitted-variants-from-file-test.properties @@ -26,15 +26,8 @@ spring.data.mongodb.database=test-db # this will be replaced with a default value "localhost" when run locally # (due to the filtering attribute specified in this project's POM) # (OR) this variable can be overridden in Gitlab with "mongo" (Gitlab services are exposed under their own hostname) -spring.data.mongodb.host=|eva.mongo.host.test| -spring.data.mongodb.password= -spring.data.mongodb.port=27017 # See https://github.com/spring-projects/spring-boot/wiki/Spring-Boot-2.1-Release-Notes#bean-overriding spring.main.allow-bean-definition-overriding=true human.mongodb.database=human-variants-test -human.mongodb.host=|eva.mongo.host.test| -human.mongodb.password= -human.mongodb.port=27017 - mongodb.read-preference=primary diff --git a/eva-accession-deprecate/src/test/resources/study-submitted-variants-test.properties b/eva-accession-deprecate/src/test/resources/study-submitted-variants-test.properties index 8dbef8788..e0e437959 100644 --- a/eva-accession-deprecate/src/test/resources/study-submitted-variants-test.properties +++ b/eva-accession-deprecate/src/test/resources/study-submitted-variants-test.properties @@ -23,15 +23,8 @@ spring.data.mongodb.database=test-db # this will be replaced with a default value "localhost" when run locally # (due to the filtering attribute specified in this project's POM) # (OR) this variable can be overridden in Gitlab with "mongo" (Gitlab services are exposed under their own hostname) -spring.data.mongodb.host=|eva.mongo.host.test| -spring.data.mongodb.password= -spring.data.mongodb.port=27017 # See https://github.com/spring-projects/spring-boot/wiki/Spring-Boot-2.1-Release-Notes#bean-overriding spring.main.allow-bean-definition-overriding=true human.mongodb.database=human-variants-test -human.mongodb.host=|eva.mongo.host.test| -human.mongodb.password= -human.mongodb.port=27017 - mongodb.read-preference=primary diff --git a/eva-accession-import-dbsnp2/pom.xml b/eva-accession-import-dbsnp2/pom.xml index ea827ba64..03889334e 100644 --- a/eva-accession-import-dbsnp2/pom.xml +++ b/eva-accession-import-dbsnp2/pom.xml @@ -2,12 +2,13 @@ + 4.0.0 + eva-accession uk.ac.ebi.eva - 0.6.57-SNAPSHOT + ${revision} - 4.0.0 eva-accession-import-dbsnp2 jar @@ -44,19 +45,28 @@ test - com.lordofthejars - nosqlunit-mongodb + org.springframework.boot + spring-boot-testcontainers + test + + + org.testcontainers + mongodb + test + + + org.testcontainers + junit-jupiter test - org.codehaus.jackson - jackson-mapper-asl - 1.9.13 + com.fasterxml.jackson.core + jackson-databind org.apache.commons commons-compress - 1.18 + 1.28.0 uk.ac.ebi.eva diff --git a/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/batch/io/DbsnpJsonClusteredVariantsWriter.java b/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/batch/io/DbsnpJsonClusteredVariantsWriter.java index 77175b2b6..1fcd0da93 100644 --- a/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/batch/io/DbsnpJsonClusteredVariantsWriter.java +++ b/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/batch/io/DbsnpJsonClusteredVariantsWriter.java @@ -18,17 +18,17 @@ import com.mongodb.MongoBulkWriteException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.batch.item.Chunk; import org.springframework.batch.item.ItemWriter; -import org.springframework.dao.DuplicateKeyException; +import org.springframework.data.mongodb.BulkOperationException; import uk.ac.ebi.ampt2d.commons.accession.core.models.EventType; - -import uk.ac.ebi.eva.accession.core.batch.io.DbsnpClusteredVariantWriter; import uk.ac.ebi.eva.accession.core.batch.io.DbsnpClusteredVariantOperationWriter; +import uk.ac.ebi.eva.accession.core.batch.io.DbsnpClusteredVariantWriter; import uk.ac.ebi.eva.accession.core.batch.io.MergeOperationBuilder; -import uk.ac.ebi.eva.accession.core.repository.nonhuman.dbsnp.DbsnpClusteredVariantAccessioningRepository; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpClusteredVariantEntity; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpClusteredVariantInactiveEntity; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpClusteredVariantOperationEntity; +import uk.ac.ebi.eva.accession.core.repository.nonhuman.dbsnp.DbsnpClusteredVariantAccessioningRepository; import uk.ac.ebi.eva.accession.core.repository.nonhuman.dbsnp.DbsnpClusteredVariantOperationRepository; import java.util.Arrays; @@ -61,21 +61,20 @@ public DbsnpJsonClusteredVariantsWriter(DbsnpClusteredVariantWriter dbsnpCluster } @Override - public void write(List clusteredVariants) throws Exception { + public void write(Chunk clusteredVariants) throws Exception { try { if (!clusteredVariants.isEmpty()) { dbsnpClusteredVariantWriter.write(clusteredVariants); - } - else { + } else { logger.warn("Could not find any clustered variants to write in the current chunk!"); } - } catch (DuplicateKeyException exception) { + } catch (BulkOperationException exception) { MongoBulkWriteException writeException = ((MongoBulkWriteException) exception.getCause()); List mergeClusteredOperations = clusteredOperationBuilder.buildMergeOperationsFromException( - (List) clusteredVariants, writeException); + List.copyOf(clusteredVariants.getItems()), writeException); if (!mergeClusteredOperations.isEmpty()) { - dbsnpClusteredVariantOperationWriter.write(mergeClusteredOperations); + dbsnpClusteredVariantOperationWriter.write(new Chunk<>(mergeClusteredOperations)); } } } @@ -86,8 +85,8 @@ private DbsnpClusteredVariantOperationEntity buildClusteredMergeOperation(DbsnpC DbsnpClusteredVariantOperationEntity operation = new DbsnpClusteredVariantOperationEntity(); operation.fill(EventType.MERGED, origin.getAccession(), mergedInto.getAccession(), - "Identical clustered variant received multiple RS identifiers", - Arrays.asList(inactiveEntity)); + "Identical clustered variant received multiple RS identifiers", + Arrays.asList(inactiveEntity)); return operation; } diff --git a/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/batch/listeners/ImportDbsnpJsonVariantsStepProgressListener.java b/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/batch/listeners/ImportDbsnpJsonVariantsStepProgressListener.java index d1879108b..e715d93ea 100644 --- a/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/batch/listeners/ImportDbsnpJsonVariantsStepProgressListener.java +++ b/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/batch/listeners/ImportDbsnpJsonVariantsStepProgressListener.java @@ -15,7 +15,7 @@ */ package uk.ac.ebi.eva.accession.dbsnp2.batch.listeners; -import org.codehaus.jackson.JsonNode; +import com.fasterxml.jackson.databind.JsonNode; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.batch.core.ExitStatus; @@ -63,8 +63,8 @@ public ExitStatus afterStep(StepExecution stepExecution) { String stepName = stepExecution.getStepName(); long numTotalItemsRead = stepExecution.getReadCount(); logger.info("Step {} finished: Items read = {}, rs written = {}, operations written = {}", - stepName, numTotalItemsRead, - importCounts.getClusteredVariantsWritten(), importCounts.getOperationsWritten()); + stepName, numTotalItemsRead, + importCounts.getClusteredVariantsWritten(), importCounts.getOperationsWritten()); // add import counts to execution context, so they can be retrieved later if the job is restarted ExecutionContext executionContext = stepExecution.getExecutionContext(); addImportCountsToExecutionContext(executionContext); diff --git a/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/batch/processors/ContigToGenbankReplacerProcessor.java b/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/batch/processors/ContigToGenbankReplacerProcessor.java index 419ca4a18..fa4986349 100644 --- a/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/batch/processors/ContigToGenbankReplacerProcessor.java +++ b/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/batch/processors/ContigToGenbankReplacerProcessor.java @@ -19,11 +19,10 @@ import org.slf4j.LoggerFactory; import org.springframework.batch.item.ItemProcessor; import uk.ac.ebi.ampt2d.commons.accession.hashing.SHA1HashingFunction; - -import uk.ac.ebi.eva.accession.core.model.ClusteredVariant; -import uk.ac.ebi.eva.accession.core.model.IClusteredVariant; import uk.ac.ebi.eva.accession.core.contig.ContigMapping; import uk.ac.ebi.eva.accession.core.contig.ContigSynonyms; +import uk.ac.ebi.eva.accession.core.model.ClusteredVariant; +import uk.ac.ebi.eva.accession.core.model.IClusteredVariant; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpClusteredVariantEntity; import uk.ac.ebi.eva.accession.core.summary.ClusteredVariantSummaryFunction; diff --git a/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/configuration/RunnerConfiguration.java b/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/configuration/RunnerConfiguration.java index da9bb9d98..23ede5cfa 100644 --- a/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/configuration/RunnerConfiguration.java +++ b/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/configuration/RunnerConfiguration.java @@ -16,16 +16,10 @@ */ package uk.ac.ebi.eva.accession.dbsnp2.configuration; -import org.springframework.batch.core.configuration.annotation.BatchConfigurer; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; - -import uk.ac.ebi.eva.commons.batch.configuration.SpringBoot1CompatibilityConfiguration; import uk.ac.ebi.eva.commons.batch.job.JobExecutionApplicationListener; -import javax.persistence.EntityManagerFactory; -import javax.sql.DataSource; - @Configuration public class RunnerConfiguration { @@ -33,11 +27,4 @@ public class RunnerConfiguration { public JobExecutionApplicationListener jobExecutionApplicationListener() { return new JobExecutionApplicationListener(); } - - @Bean - public BatchConfigurer configurer(DataSource dataSource, EntityManagerFactory entityManagerFactory) - throws Exception { - return SpringBoot1CompatibilityConfiguration.getSpringBoot1CompatibleBatchConfigurer(dataSource, - entityManagerFactory); - } } diff --git a/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/configuration/batch/io/ImportDbsnpJsonVariantsWriterConfiguration.java b/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/configuration/batch/io/ImportDbsnpJsonVariantsWriterConfiguration.java index 4f78fc659..110c7efba 100644 --- a/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/configuration/batch/io/ImportDbsnpJsonVariantsWriterConfiguration.java +++ b/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/configuration/batch/io/ImportDbsnpJsonVariantsWriterConfiguration.java @@ -24,12 +24,12 @@ import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Import; import org.springframework.data.mongodb.core.MongoTemplate; -import uk.ac.ebi.eva.accession.core.configuration.nonhuman.MongoConfiguration; import uk.ac.ebi.eva.accession.core.batch.io.DbsnpClusteredVariantOperationWriter; import uk.ac.ebi.eva.accession.core.batch.io.DbsnpClusteredVariantWriter; import uk.ac.ebi.eva.accession.core.batch.listeners.ImportCounts; -import uk.ac.ebi.eva.accession.core.repository.nonhuman.dbsnp.DbsnpClusteredVariantAccessioningRepository; +import uk.ac.ebi.eva.accession.core.configuration.nonhuman.MongoConfiguration; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpClusteredVariantEntity; +import uk.ac.ebi.eva.accession.core.repository.nonhuman.dbsnp.DbsnpClusteredVariantAccessioningRepository; import uk.ac.ebi.eva.accession.core.repository.nonhuman.dbsnp.DbsnpClusteredVariantOperationRepository; import uk.ac.ebi.eva.accession.dbsnp2.batch.io.DbsnpJsonClusteredVariantsWriter; import uk.ac.ebi.eva.accession.dbsnp2.parameters.InputParameters; diff --git a/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/configuration/batch/jobs/ImportDbsnpJsonVariantsJobConfiguration.java b/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/configuration/batch/jobs/ImportDbsnpJsonVariantsJobConfiguration.java index 000dca4aa..d747cfa86 100644 --- a/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/configuration/batch/jobs/ImportDbsnpJsonVariantsJobConfiguration.java +++ b/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/configuration/batch/jobs/ImportDbsnpJsonVariantsJobConfiguration.java @@ -17,9 +17,10 @@ import org.springframework.batch.core.Job; import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; -import org.springframework.batch.core.configuration.annotation.JobBuilderFactory; +import org.springframework.batch.core.job.builder.JobBuilder; import org.springframework.batch.core.job.flow.Flow; import org.springframework.batch.core.launch.support.RunIdIncrementer; +import org.springframework.batch.core.repository.JobRepository; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; @@ -40,8 +41,8 @@ public class ImportDbsnpJsonVariantsJobConfiguration { private Flow importFlow; @Bean(IMPORT_DBSNP_JSON_VARIANTS_JOB) - public Job importDbsnpJsonVariantsJob(JobBuilderFactory jobBuilderFactory) { - return jobBuilderFactory.get(IMPORT_DBSNP_JSON_VARIANTS_JOB) + public Job importDbsnpJsonVariantsJob(JobRepository jobRepository) { + return new JobBuilder(IMPORT_DBSNP_JSON_VARIANTS_JOB, jobRepository) .incrementer(new RunIdIncrementer()) .start(importFlow) .end() diff --git a/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/configuration/batch/steps/ImportDbsnpJsonVariantsStepConfiguration.java b/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/configuration/batch/steps/ImportDbsnpJsonVariantsStepConfiguration.java index d53f3e607..0828b2229 100644 --- a/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/configuration/batch/steps/ImportDbsnpJsonVariantsStepConfiguration.java +++ b/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/configuration/batch/steps/ImportDbsnpJsonVariantsStepConfiguration.java @@ -19,7 +19,8 @@ import org.springframework.batch.core.Step; import org.springframework.batch.core.StepExecutionListener; import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; -import org.springframework.batch.core.configuration.annotation.StepBuilderFactory; +import org.springframework.batch.core.repository.JobRepository; +import org.springframework.batch.core.step.builder.StepBuilder; import org.springframework.batch.item.ItemProcessor; import org.springframework.batch.item.ItemWriter; import org.springframework.batch.item.file.FlatFileItemReader; @@ -28,13 +29,14 @@ import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.transaction.PlatformTransactionManager; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpClusteredVariantEntity; -import static uk.ac.ebi.eva.accession.dbsnp2.configuration.BeanNames.DBSNP_JSON_VARIANT_READER; import static uk.ac.ebi.eva.accession.dbsnp2.configuration.BeanNames.DBSNP_JSON_VARIANT_PROCESSOR; +import static uk.ac.ebi.eva.accession.dbsnp2.configuration.BeanNames.DBSNP_JSON_VARIANT_READER; import static uk.ac.ebi.eva.accession.dbsnp2.configuration.BeanNames.DBSNP_JSON_VARIANT_WRITER; -import static uk.ac.ebi.eva.accession.dbsnp2.configuration.BeanNames.IMPORT_DBSNP_JSON_VARIANTS_STEP; import static uk.ac.ebi.eva.accession.dbsnp2.configuration.BeanNames.IMPORT_DBSNP_JSON_VARIANTS_PROGRESS_LISTENER; +import static uk.ac.ebi.eva.accession.dbsnp2.configuration.BeanNames.IMPORT_DBSNP_JSON_VARIANTS_STEP; /** * Configuration for dbSNP JSON import flow step @@ -61,14 +63,14 @@ public class ImportDbsnpJsonVariantsStepConfiguration { @Bean(IMPORT_DBSNP_JSON_VARIANTS_STEP) - public Step importDbsnpJsonVariantsStep(StepBuilderFactory stepBuilderFactory, + public Step importDbsnpJsonVariantsStep(JobRepository jobRepository, PlatformTransactionManager transactionManager, SimpleCompletionPolicy chunkSizeCompletionPolicy) { - return stepBuilderFactory.get(IMPORT_DBSNP_JSON_VARIANTS_STEP) - .chunk(chunkSizeCompletionPolicy) - .reader(variantReader) - .processor(variantProcessor) - .writer(variantWriter) - .listener(importDbsnpJsonVariantsProgressListener) - .build(); + return new StepBuilder(IMPORT_DBSNP_JSON_VARIANTS_STEP, jobRepository) + .chunk(chunkSizeCompletionPolicy, transactionManager) + .reader(variantReader) + .processor(variantProcessor) + .writer(variantWriter) + .listener(importDbsnpJsonVariantsProgressListener) + .build(); } } diff --git a/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/runner/DbsnpJsonImportVariantsJobLauncherCommandLineRunner.java b/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/runner/DbsnpJsonImportVariantsJobLauncherCommandLineRunner.java index 198f4246a..ff80ea812 100644 --- a/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/runner/DbsnpJsonImportVariantsJobLauncherCommandLineRunner.java +++ b/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/runner/DbsnpJsonImportVariantsJobLauncherCommandLineRunner.java @@ -24,7 +24,6 @@ import org.springframework.batch.core.JobParametersInvalidException; import org.springframework.batch.core.explore.JobExplorer; import org.springframework.batch.core.launch.JobLauncher; -import org.springframework.batch.core.launch.JobParametersNotFoundException; import org.springframework.batch.core.repository.JobExecutionAlreadyRunningException; import org.springframework.batch.core.repository.JobInstanceAlreadyCompleteException; import org.springframework.batch.core.repository.JobRepository; @@ -32,11 +31,10 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.ExitCodeGenerator; -import org.springframework.boot.autoconfigure.batch.JobLauncherCommandLineRunner; +import org.springframework.boot.autoconfigure.batch.JobLauncherApplicationRunner; import org.springframework.context.ApplicationEventPublisherAware; import org.springframework.stereotype.Component; import org.springframework.util.PatternMatchUtils; - import uk.ac.ebi.eva.accession.core.runner.CommandLineRunnerUtils; import uk.ac.ebi.eva.accession.dbsnp2.parameters.InputParameters; import uk.ac.ebi.eva.commons.batch.exception.NoJobToExecuteException; @@ -51,7 +49,7 @@ * Custom job launcher command line runner to integrate Job with Input Parameters */ @Component -public class DbsnpJsonImportVariantsJobLauncherCommandLineRunner extends JobLauncherCommandLineRunner implements +public class DbsnpJsonImportVariantsJobLauncherCommandLineRunner extends JobLauncherApplicationRunner implements ApplicationEventPublisherAware, ExitCodeGenerator { private static final Logger logger = LoggerFactory @@ -81,7 +79,7 @@ public class DbsnpJsonImportVariantsJobLauncherCommandLineRunner extends JobLaun private boolean abnormalExit; public DbsnpJsonImportVariantsJobLauncherCommandLineRunner(JobLauncher jobLauncher, JobExplorer jobExplorer, - JobRepository jobRepository) { + JobRepository jobRepository) { super(jobLauncher, jobExplorer, jobRepository); this.jobExplorer = jobExplorer; this.jobRepository = jobRepository; @@ -94,9 +92,9 @@ public void setJobs(Collection jobs) { } @Override - public void setJobNames(String jobName) { + public void setJobName(String jobName) { this.jobName = jobName; - super.setJobNames(jobName); + super.setJobName(jobName); } @Override @@ -119,16 +117,15 @@ public void run(String... args) throws JobExecutionException { JobStatusManager.checkIfPropertiesHaveBeenProvided(jobParameters); if (inputParameters.isForceRestart()) { JobExecution previousJobExecution = CommandLineRunnerUtils.getLastJobExecution(jobName, jobExplorer, - jobParameters); + jobParameters); CommandLineRunnerUtils.markPreviousJobAsFailed(jobName, jobRepository, previousJobExecution != null ? previousJobExecution.getJobParameters() : jobParameters); - } - else { + } else { jobParameters = CommandLineRunnerUtils.addRunIDToJobParameters(jobName, jobExplorer, jobParameters); } launchJob(jobParameters); } catch (NoPreviousJobExecutionException | NoParametersHaveBeenPassedException | NoJobToExecuteException - | JobParametersInvalidException | JobExecutionAlreadyRunningException e) { + | JobParametersInvalidException | JobExecutionAlreadyRunningException e) { logger.error(e.getMessage()); logger.debug("Error trace", e); abnormalExit = true; @@ -147,8 +144,7 @@ private void launchJob(JobParameters jobParameters) throws JobExecutionException @Override protected void execute(Job job, JobParameters jobParameters) throws JobExecutionAlreadyRunningException, - JobRestartException, JobInstanceAlreadyCompleteException, JobParametersInvalidException, - JobParametersNotFoundException { + JobRestartException, JobInstanceAlreadyCompleteException, JobParametersInvalidException { logger.info("Running job '" + jobName + "' with parameters: " + jobParameters); super.execute(job, jobParameters); } diff --git a/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/batch/io/DbsnpJsonClusteredVariantsWriterTest.java b/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/batch/io/DbsnpJsonClusteredVariantsWriterTest.java index 399fd2382..8c2b413dd 100644 --- a/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/batch/io/DbsnpJsonClusteredVariantsWriterTest.java +++ b/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/batch/io/DbsnpJsonClusteredVariantsWriterTest.java @@ -15,9 +15,10 @@ */ package uk.ac.ebi.eva.accession.dbsnp2.batch.io; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.batch.item.Chunk; import org.springframework.batch.test.StepScopeTestExecutionListener; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.mongodb.core.MongoTemplate; @@ -25,22 +26,24 @@ import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestExecutionListeners; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import org.springframework.test.context.support.DependencyInjectionTestExecutionListener; import uk.ac.ebi.ampt2d.commons.accession.core.models.EventType; import uk.ac.ebi.ampt2d.commons.accession.hashing.SHA1HashingFunction; -import uk.ac.ebi.eva.accession.core.model.ClusteredVariant; -import uk.ac.ebi.eva.accession.core.model.IClusteredVariant; import uk.ac.ebi.eva.accession.core.batch.io.DbsnpClusteredVariantOperationWriter; import uk.ac.ebi.eva.accession.core.batch.io.DbsnpClusteredVariantWriter; import uk.ac.ebi.eva.accession.core.batch.listeners.ImportCounts; -import uk.ac.ebi.eva.accession.core.repository.nonhuman.dbsnp.DbsnpClusteredVariantAccessioningRepository; +import uk.ac.ebi.eva.accession.core.model.ClusteredVariant; +import uk.ac.ebi.eva.accession.core.model.IClusteredVariant; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpClusteredVariantEntity; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpClusteredVariantInactiveEntity; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpClusteredVariantOperationEntity; +import uk.ac.ebi.eva.accession.core.repository.nonhuman.dbsnp.DbsnpClusteredVariantAccessioningRepository; import uk.ac.ebi.eva.accession.core.repository.nonhuman.dbsnp.DbsnpClusteredVariantOperationRepository; import uk.ac.ebi.eva.accession.core.summary.ClusteredVariantSummaryFunction; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.dbsnp2.test.BatchTestConfiguration; +import uk.ac.ebi.eva.accession.dbsnp2.test.MongoTestConfiguration; import uk.ac.ebi.eva.commons.core.models.VariantType; import java.time.LocalDateTime; @@ -49,15 +52,15 @@ import java.util.List; import java.util.function.Function; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; -@RunWith(SpringRunner.class) -@ContextConfiguration(classes = {BatchTestConfiguration.class}) +@ExtendWith(SpringExtension.class) +@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class}) @TestExecutionListeners({DependencyInjectionTestExecutionListener.class, StepScopeTestExecutionListener.class}) @TestPropertySource({"classpath:application.properties"}) -public class DbsnpJsonClusteredVariantsWriterTest { +public class DbsnpJsonClusteredVariantsWriterTest extends MongoTestContainerHelper { private DbsnpJsonClusteredVariantsWriter dbsnpJsonClusteredVariantsWriter; @@ -76,91 +79,91 @@ public class DbsnpJsonClusteredVariantsWriterTest { private DbsnpClusteredVariantEntity variantEntity2; private Function hashingFuncClustered = - new ClusteredVariantSummaryFunction().andThen(new SHA1HashingFunction()); + new ClusteredVariantSummaryFunction().andThen(new SHA1HashingFunction()); - @Before + @BeforeEach public void setUp() { importCounts = new ImportCounts(); DbsnpClusteredVariantWriter dbsnpClusteredVariantWriter = new DbsnpClusteredVariantWriter(mongoTemplate, - importCounts); + importCounts); DbsnpClusteredVariantOperationWriter dbsnpClusteredVariantOperationWriter = new DbsnpClusteredVariantOperationWriter(mongoTemplate, importCounts); variantEntity1 = buildClusteredVariantEntity(1L, - buildClusteredVariant("acsn1", - "contig1", - 1L, - VariantType.SNV)); + buildClusteredVariant("acsn1", + "contig1", + 1L, + VariantType.SNV)); dbsnpJsonClusteredVariantsWriter = new DbsnpJsonClusteredVariantsWriter(dbsnpClusteredVariantWriter, dbsnpClusteredVariantOperationWriter, - dbsnpClusteredVariantOperationRepository, - dbsnpClusteredVariantAccessioningRepository); + dbsnpClusteredVariantOperationRepository, + dbsnpClusteredVariantAccessioningRepository); mongoTemplate.dropCollection(DbsnpClusteredVariantEntity.class); mongoTemplate.dropCollection(DbsnpClusteredVariantOperationEntity.class); } @Test public void writeSingleVariant() throws Exception { - dbsnpJsonClusteredVariantsWriter.write(Collections.singletonList(variantEntity1)); + dbsnpJsonClusteredVariantsWriter.write(Chunk.of(variantEntity1)); assertClusteredVariantStored(1, Collections.singletonList(variantEntity1)); } @Test public void writeMultipleVariantsWithDifferentContig() throws Exception { variantEntity2 = buildClusteredVariantEntity(variantEntity1.getAccession(), - buildClusteredVariant(variantEntity1.getAssemblyAccession(), - "contig2", - variantEntity1.getStart(), - variantEntity1.getType())); + buildClusteredVariant(variantEntity1.getAssemblyAccession(), + "contig2", + variantEntity1.getStart(), + variantEntity1.getType())); List clusteredVariantEntities = Arrays.asList(variantEntity1, variantEntity2); - dbsnpJsonClusteredVariantsWriter.write(clusteredVariantEntities); + dbsnpJsonClusteredVariantsWriter.write(new Chunk<>(clusteredVariantEntities)); assertClusteredVariantStored(2, clusteredVariantEntities); } @Test public void writeMultipleVariantsWithDifferentStart() throws Exception { variantEntity2 = buildClusteredVariantEntity(variantEntity1.getAccession(), - buildClusteredVariant(variantEntity1.getAssemblyAccession(), - variantEntity1.getContig(), - 2L, - variantEntity1.getType())); + buildClusteredVariant(variantEntity1.getAssemblyAccession(), + variantEntity1.getContig(), + 2L, + variantEntity1.getType())); List clusteredVariantEntities = Arrays.asList(variantEntity1, variantEntity2); - dbsnpJsonClusteredVariantsWriter.write(clusteredVariantEntities); + dbsnpJsonClusteredVariantsWriter.write(new Chunk<>(clusteredVariantEntities)); assertClusteredVariantStored(2, clusteredVariantEntities); } @Test public void writeMultipleVariantsWithDifferentVariantType() throws Exception { variantEntity2 = buildClusteredVariantEntity(variantEntity1.getAccession(), - buildClusteredVariant(variantEntity1.getAssemblyAccession(), - variantEntity1.getContig(), - variantEntity1.getStart(), - VariantType.DEL)); + buildClusteredVariant(variantEntity1.getAssemblyAccession(), + variantEntity1.getContig(), + variantEntity1.getStart(), + VariantType.DEL)); List clusteredVariantEntities = Arrays.asList(variantEntity1, variantEntity2); - dbsnpJsonClusteredVariantsWriter.write(clusteredVariantEntities); + dbsnpJsonClusteredVariantsWriter.write(new Chunk<>(clusteredVariantEntities)); assertClusteredVariantStored(2, clusteredVariantEntities); } @Test public void writeMultipleVariantsWithDifferentAssemblyAccession() throws Exception { variantEntity2 = buildClusteredVariantEntity(variantEntity1.getAccession(), - buildClusteredVariant("acsn2", - variantEntity1.getContig(), - variantEntity1.getStart(), - variantEntity1.getType())); + buildClusteredVariant("acsn2", + variantEntity1.getContig(), + variantEntity1.getStart(), + variantEntity1.getType())); List clusteredVariantEntities = Arrays.asList(variantEntity1, variantEntity2); - dbsnpJsonClusteredVariantsWriter.write(clusteredVariantEntities); + dbsnpJsonClusteredVariantsWriter.write(new Chunk<>(clusteredVariantEntities)); assertClusteredVariantStored(2, clusteredVariantEntities); } @Test public void writeMultipleVariantsWithDifferentRsIDWithSameClusteredVariant() throws Exception { variantEntity2 = buildClusteredVariantEntity(2L, - variantEntity1.getModel()); + variantEntity1.getModel()); DbsnpClusteredVariantEntity variantEntity3 = buildClusteredVariantEntity(3L, - variantEntity1.getModel()); + variantEntity1.getModel()); List clusteredVariantEntities = Arrays.asList(variantEntity1, variantEntity2, - variantEntity3); - dbsnpJsonClusteredVariantsWriter.write(clusteredVariantEntities); + variantEntity3); + dbsnpJsonClusteredVariantsWriter.write(new Chunk<>(clusteredVariantEntities)); assertClusteredVariantStored(1, clusteredVariantEntities); DbsnpClusteredVariantOperationEntity clusteredVariantOperationEntity1 = @@ -168,21 +171,21 @@ public void writeMultipleVariantsWithDifferentRsIDWithSameClusteredVariant() thr DbsnpClusteredVariantOperationEntity clusteredVariantOperationEntity2 = new DbsnpClusteredVariantOperationEntity(); clusteredVariantOperationEntity1.fill(EventType.MERGED, variantEntity2.getAccession(), - variantEntity1.getAccession(), - "Identical clustered variant received multiple RS identifiers", - Arrays.asList(new DbsnpClusteredVariantInactiveEntity(variantEntity2))); + variantEntity1.getAccession(), + "Identical clustered variant received multiple RS identifiers", + Arrays.asList(new DbsnpClusteredVariantInactiveEntity(variantEntity2))); clusteredVariantOperationEntity2.fill(EventType.MERGED, variantEntity3.getAccession(), - variantEntity1.getAccession(), - "Identical clustered variant received multiple RS identifiers", - Arrays.asList(new DbsnpClusteredVariantInactiveEntity(variantEntity3))); + variantEntity1.getAccession(), + "Identical clustered variant received multiple RS identifiers", + Arrays.asList(new DbsnpClusteredVariantInactiveEntity(variantEntity3))); assertClusteredVariantOperationStored(2, Arrays.asList(clusteredVariantOperationEntity1, - clusteredVariantOperationEntity2)); + clusteredVariantOperationEntity2)); } @Test public void writeDuplicateVariantShouldNotBeStored() throws Exception { List clusteredVariantEntities = Arrays.asList(variantEntity1, variantEntity1); - dbsnpJsonClusteredVariantsWriter.write(clusteredVariantEntities); + dbsnpJsonClusteredVariantsWriter.write(new Chunk<>(clusteredVariantEntities)); assertClusteredVariantStored(1, clusteredVariantEntities); } @@ -193,22 +196,22 @@ private DbsnpClusteredVariantEntity buildClusteredVariantEntity(Long accession, private IClusteredVariant buildClusteredVariant(String accession, String contig, long start, VariantType variantType) { return new ClusteredVariant(accession, - 9606, - contig, - start, - variantType, - Boolean.FALSE, - LocalDateTime.now()); + 9606, + contig, + start, + variantType, + Boolean.FALSE, + LocalDateTime.now()); } private void assertClusteredVariantStored(int expectedVariants, List clusteredVariantEntities) { List actualClusteredVariantEntities = mongoTemplate.find - (new Query(), DbsnpClusteredVariantEntity.class); + (new Query(), DbsnpClusteredVariantEntity.class); assertEquals(expectedVariants, actualClusteredVariantEntities.size()); assertEquals(expectedVariants, importCounts.getClusteredVariantsWritten()); clusteredVariantEntities.forEach(entity -> - assertTrue(actualClusteredVariantEntities.contains(entity))); + assertTrue(actualClusteredVariantEntities.contains(entity))); } private void assertClusteredVariantOperationStored @@ -223,7 +226,7 @@ private void assertClusteredVariantStored(int expectedVariants, } private boolean dbsnpClusteredVariantOperationEntitiesEqual(DbsnpClusteredVariantOperationEntity entity1, - DbsnpClusteredVariantOperationEntity entity2) { + DbsnpClusteredVariantOperationEntity entity2) { return entity1.getAccession().equals(entity2.getAccession()) && entity1.getMergedInto().equals(entity2.getMergedInto()) && entity1.getInactiveObjects().containsAll(entity2.getInactiveObjects()) && diff --git a/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/batch/io/DbsnpJsonItemReaderTest.java b/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/batch/io/DbsnpJsonItemReaderTest.java index 3e6f60b98..bd95e7e1d 100644 --- a/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/batch/io/DbsnpJsonItemReaderTest.java +++ b/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/batch/io/DbsnpJsonItemReaderTest.java @@ -16,10 +16,8 @@ package uk.ac.ebi.eva.accession.dbsnp2.batch.io; import com.fasterxml.jackson.databind.JsonNode; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.batch.item.ExecutionContext; import org.springframework.batch.item.ItemStreamException; import org.springframework.batch.item.file.FlatFileItemReader; @@ -29,7 +27,7 @@ import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestExecutionListeners; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import org.springframework.test.context.support.DependencyInjectionTestExecutionListener; import uk.ac.ebi.eva.accession.dbsnp2.test.BatchTestConfiguration; @@ -37,11 +35,12 @@ import java.util.ArrayList; import java.util.List; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; import static uk.ac.ebi.eva.accession.dbsnp2.configuration.BeanNames.DBSNP_JSON_VARIANT_READER; -@RunWith(SpringRunner.class) +@ExtendWith(SpringExtension.class) @ContextConfiguration(classes = {BatchTestConfiguration.class}) @TestExecutionListeners({DependencyInjectionTestExecutionListener.class, StepScopeTestExecutionListener.class}) @@ -52,13 +51,10 @@ public class DbsnpJsonItemReaderTest { @Qualifier(DBSNP_JSON_VARIANT_READER) private FlatFileItemReader reader; - @Rule - public ExpectedException thrown = ExpectedException.none(); - @Test public void readExistingSource() throws Exception { reader.setResource(new BzipLazyResource( - new File("src/test/resources/input-files/test-dbsnp.json.bz2"))); + new File("src/test/resources/input-files/test-dbsnp.json.bz2"))); reader.open(new ExecutionContext()); List variants = readAll(reader); assertEquals(26, variants.size()); @@ -67,8 +63,7 @@ public void readExistingSource() throws Exception { @Test public void readNonExistingSource() throws Exception { reader.setResource(new BzipLazyResource(new File("INVALID_DIRECTORY"))); - thrown.expect(ItemStreamException.class); - reader.open(new ExecutionContext()); + assertThrows(ItemStreamException.class, () -> reader.open(new ExecutionContext())); } private List readAll(FlatFileItemReader reader) throws Exception { diff --git a/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/batch/processors/ContigToGenbankReplacerProcessorTest.java b/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/batch/processors/ContigToGenbankReplacerProcessorTest.java index 8de028099..253688b54 100644 --- a/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/batch/processors/ContigToGenbankReplacerProcessorTest.java +++ b/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/batch/processors/ContigToGenbankReplacerProcessorTest.java @@ -15,15 +15,12 @@ */ package uk.ac.ebi.eva.accession.dbsnp2.batch.processors; -import org.junit.Before; -import org.junit.Rule; -import org.junit.rules.ExpectedException; -import org.junit.Test; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import uk.ac.ebi.ampt2d.commons.accession.hashing.SHA1HashingFunction; - +import uk.ac.ebi.eva.accession.core.contig.ContigMapping; import uk.ac.ebi.eva.accession.core.model.ClusteredVariant; import uk.ac.ebi.eva.accession.core.model.IClusteredVariant; -import uk.ac.ebi.eva.accession.core.contig.ContigMapping; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpClusteredVariantEntity; import uk.ac.ebi.eva.accession.core.summary.ClusteredVariantSummaryFunction; import uk.ac.ebi.eva.commons.core.models.VariantType; @@ -31,7 +28,8 @@ import java.time.LocalDateTime; import java.util.function.Function; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; public class ContigToGenbankReplacerProcessorTest { @@ -39,11 +37,8 @@ public class ContigToGenbankReplacerProcessorTest { private Function hashingFunction = new ClusteredVariantSummaryFunction().andThen(new SHA1HashingFunction()); - - @Rule - public ExpectedException thrown = ExpectedException.none(); - @Before + @BeforeEach public void setUp() throws Exception { String fileString = ContigToGenbankReplacerProcessorTest.class.getResource( "/input-files/GCF_000001405.38_GRCh38.p12_assembly_report.txt").toString(); @@ -53,19 +48,19 @@ public void setUp() throws Exception { } @Test - public void contigGenbank() throws Exception { + public void contigGenbank() { DbsnpClusteredVariantEntity variant = buildMockVariant("CM000686.2"); assertEquals("CM000686.2", processor.process(variant).getContig()); } @Test - public void contigChrToGenbank() throws Exception { + public void contigChrToGenbank() { DbsnpClusteredVariantEntity variant = buildMockVariant("chrY"); assertEquals("CM000686.2", processor.process(variant).getContig()); } @Test - public void contigRefseqToGenbank() throws Exception { + public void contigRefseqToGenbank() { DbsnpClusteredVariantEntity variant = buildMockVariant("NC_000024.10"); assertEquals("CM000686.2", processor.process(variant).getContig()); } @@ -73,71 +68,69 @@ public void contigRefseqToGenbank() throws Exception { @Test public void genbankAndRefseqNotEquivalents() throws Exception { DbsnpClusteredVariantEntity variant = buildMockVariant("chr3"); - thrown.expect(IllegalStateException.class); - assertEquals("chr3", processor.process(variant).getContig()); + assertThrows(IllegalStateException.class, () -> processor.process(variant)); } @Test - public void genbankAndRefseqNotEquivalentsRefseqNotPresent() throws Exception { + public void genbankAndRefseqNotEquivalentsRefseqNotPresent() { DbsnpClusteredVariantEntity variant = buildMockVariant("chrUn_KI270752v1"); assertEquals("KI270752.1", processor.process(variant).getContig()); } @Test - public void genbankAndRefseqNotEquivalentsGenbankNotPresent() throws Exception { + public void genbankAndRefseqNotEquivalentsGenbankNotPresent() { DbsnpClusteredVariantEntity variant = buildMockVariant("tstchr2"); assertEquals("tstchr2", processor.process(variant).getContig()); } @Test - public void genbankAndRefseqNotEquivalentsNonePresent() throws Exception { + public void genbankAndRefseqNotEquivalentsNonePresent() { DbsnpClusteredVariantEntity variant = buildMockVariant("tstchr3"); assertEquals("tstchr3", processor.process(variant).getContig()); } @Test - public void contigNotFoundInAssemblyReport() throws Exception { + public void contigNotFoundInAssemblyReport() { DbsnpClusteredVariantEntity variant = buildMockVariant("chr"); - thrown.expect(IllegalStateException.class); - assertEquals("chr", processor.process(variant).getContig()); + assertThrows(IllegalStateException.class, () -> processor.process(variant)); } @Test - public void noGenbankDontConvert() throws Exception { + public void noGenbankDontConvert() { DbsnpClusteredVariantEntity variant = buildMockVariant("tstchr4"); assertEquals("tstchr4", processor.process(variant).getContig()); } @Test - public void updatedHashedMessageAfterContigReplacement() throws Exception { + public void updatedHashedMessageAfterContigReplacement() { DbsnpClusteredVariantEntity processedVariant = processor.process(buildMockVariant("NC_000024.10")); assertEquals(new SHA1HashingFunction().apply("1_CM000686.2_1_SNV"), - processedVariant.getHashedMessage()); + processedVariant.getHashedMessage()); processedVariant = processor.process(buildMockVariant("chrY")); assertEquals(new SHA1HashingFunction().apply("1_CM000686.2_1_SNV"), - processedVariant.getHashedMessage()); + processedVariant.getHashedMessage()); processedVariant = processor.process(buildMockVariant("chrUn_KI270752v1")); assertEquals(new SHA1HashingFunction().apply("1_KI270752.1_1_SNV"), - processedVariant.getHashedMessage()); + processedVariant.getHashedMessage()); processedVariant = processor.process(buildMockVariant("tstchr2")); assertEquals(new SHA1HashingFunction().apply("1_tstchr2_1_SNV"), - processedVariant.getHashedMessage()); + processedVariant.getHashedMessage()); } private DbsnpClusteredVariantEntity buildMockVariant(String originalChromosome) { ClusteredVariant newVariant = new ClusteredVariant("1", - 9606, - originalChromosome, - 1, - VariantType.SNV, - Boolean.FALSE, - LocalDateTime.now()); + 9606, + originalChromosome, + 1, + VariantType.SNV, + Boolean.FALSE, + LocalDateTime.now()); return new DbsnpClusteredVariantEntity(1L, - hashingFunction.apply(newVariant), - newVariant, - 1); + hashingFunction.apply(newVariant), + newVariant, + 1); } } diff --git a/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/batch/processors/JsonNodeToClusteredVariantProcessorTest.java b/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/batch/processors/JsonNodeToClusteredVariantProcessorTest.java index 2cb816149..0b8e3f97a 100644 --- a/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/batch/processors/JsonNodeToClusteredVariantProcessorTest.java +++ b/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/batch/processors/JsonNodeToClusteredVariantProcessorTest.java @@ -16,11 +16,9 @@ package uk.ac.ebi.eva.accession.dbsnp2.batch.processors; import com.fasterxml.jackson.databind.JsonNode; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.batch.item.ExecutionContext; import org.springframework.batch.item.file.FlatFileItemReader; import org.springframework.batch.test.StepScopeTestExecutionListener; @@ -29,12 +27,11 @@ import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestExecutionListeners; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import org.springframework.test.context.support.DependencyInjectionTestExecutionListener; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpClusteredVariantEntity; import uk.ac.ebi.eva.accession.dbsnp2.batch.io.BzipLazyResource; import uk.ac.ebi.eva.accession.dbsnp2.batch.io.JsonNodeLineMapper; -import uk.ac.ebi.eva.accession.dbsnp2.batch.processors.JsonNodeToClusteredVariantProcessor; import uk.ac.ebi.eva.accession.dbsnp2.parameters.InputParameters; import uk.ac.ebi.eva.accession.dbsnp2.test.BatchTestConfiguration; import uk.ac.ebi.eva.commons.core.models.VariantType; @@ -45,11 +42,11 @@ import java.util.Objects; import java.util.stream.Collectors; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; import static uk.ac.ebi.eva.accession.dbsnp2.configuration.BeanNames.DBSNP_JSON_VARIANT_READER; -@RunWith(SpringRunner.class) +@ExtendWith(SpringExtension.class) @ContextConfiguration(classes = {BatchTestConfiguration.class}) @TestExecutionListeners({DependencyInjectionTestExecutionListener.class, StepScopeTestExecutionListener.class}) @@ -63,10 +60,8 @@ public class JsonNodeToClusteredVariantProcessorTest { private InputParameters inputParameters; private JsonNodeToClusteredVariantProcessor processor; private List variants = new ArrayList<>(); - @Rule - public ExpectedException thrown = ExpectedException.none(); - @Before + @BeforeEach public void setUp() throws Exception { reader.setResource(new BzipLazyResource( new File("src/test/resources/input-files/test-dbsnp.json.bz2"))); diff --git a/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/configuration/jobs/ImportDbsnpJsonVariantsJobConfigurationTest.java b/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/configuration/jobs/ImportDbsnpJsonVariantsJobConfigurationTest.java index 64f02cbde..eb4fcded3 100644 --- a/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/configuration/jobs/ImportDbsnpJsonVariantsJobConfigurationTest.java +++ b/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/configuration/jobs/ImportDbsnpJsonVariantsJobConfigurationTest.java @@ -15,37 +15,44 @@ */ package uk.ac.ebi.eva.accession.dbsnp2.configuration.jobs; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.batch.core.BatchStatus; import org.springframework.batch.core.JobExecution; import org.springframework.batch.core.StepExecution; import org.springframework.batch.test.JobLauncherTestUtils; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; - +import org.springframework.test.context.junit.jupiter.SpringExtension; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.dbsnp2.parameters.InputParameters; +import uk.ac.ebi.eva.accession.dbsnp2.test.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.accession.dbsnp2.test.BatchTestConfiguration; +import uk.ac.ebi.eva.accession.dbsnp2.test.MongoTestConfiguration; import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.stream.Collectors; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import static uk.ac.ebi.eva.accession.dbsnp2.configuration.BeanNames.IMPORT_DBSNP_JSON_VARIANTS_STEP; +import static uk.ac.ebi.eva.accession.dbsnp2.test.BatchTestConfiguration.JOB_IMPORT_DBSNP_JSON_VARIANTS_JOB; -@RunWith(SpringRunner.class) -@ContextConfiguration(classes = {BatchTestConfiguration.class}) +@ExtendWith(SpringExtension.class) +@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class, + BatchJobRepositoryTestConfiguration.class}) @TestPropertySource("classpath:application.properties") -public class ImportDbsnpJsonVariantsJobConfigurationTest { +public class ImportDbsnpJsonVariantsJobConfigurationTest extends MongoTestContainerHelper { @Autowired private InputParameters inputParameters; + @Autowired + @Qualifier(JOB_IMPORT_DBSNP_JSON_VARIANTS_JOB) private JobLauncherTestUtils jobLauncherTestUtils; @Test diff --git a/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/configuration/steps/ImportDbsnpJsonVariantsStepConfigurationTest.java b/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/configuration/steps/ImportDbsnpJsonVariantsStepConfigurationTest.java index bcf488998..06522e338 100644 --- a/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/configuration/steps/ImportDbsnpJsonVariantsStepConfigurationTest.java +++ b/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/configuration/steps/ImportDbsnpJsonVariantsStepConfigurationTest.java @@ -15,36 +15,44 @@ */ package uk.ac.ebi.eva.accession.dbsnp2.configuration.steps; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.batch.core.BatchStatus; import org.springframework.batch.core.JobExecution; import org.springframework.batch.test.JobLauncherTestUtils; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.data.mongodb.core.query.Query; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpClusteredVariantEntity; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; +import uk.ac.ebi.eva.accession.dbsnp2.test.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.accession.dbsnp2.test.BatchTestConfiguration; +import uk.ac.ebi.eva.accession.dbsnp2.test.MongoTestConfiguration; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import static uk.ac.ebi.eva.accession.dbsnp2.configuration.BeanNames.IMPORT_DBSNP_JSON_VARIANTS_STEP; +import static uk.ac.ebi.eva.accession.dbsnp2.test.BatchTestConfiguration.JOB_IMPORT_DBSNP_JSON_VARIANTS_JOB; -@RunWith(SpringRunner.class) -@ContextConfiguration(classes = {BatchTestConfiguration.class}) +@ExtendWith(SpringExtension.class) +@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class, + BatchJobRepositoryTestConfiguration.class}) @TestPropertySource("classpath:application.properties") -public class ImportDbsnpJsonVariantsStepConfigurationTest { +public class ImportDbsnpJsonVariantsStepConfigurationTest extends MongoTestContainerHelper { @Autowired + @Qualifier(JOB_IMPORT_DBSNP_JSON_VARIANTS_JOB) private JobLauncherTestUtils jobLauncherTestUtils; + @Autowired private MongoTemplate mongoTemplate; - @Before + @BeforeEach public void setUp() { mongoTemplate.dropCollection(DbsnpClusteredVariantEntity.class); } diff --git a/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/runner/DbsnpJsonImportVariantsJobLauncherCommandLineRunnerTest.java b/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/runner/DbsnpJsonImportVariantsJobLauncherCommandLineRunnerTest.java index be2692ac6..3add32b31 100644 --- a/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/runner/DbsnpJsonImportVariantsJobLauncherCommandLineRunnerTest.java +++ b/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/runner/DbsnpJsonImportVariantsJobLauncherCommandLineRunnerTest.java @@ -16,12 +16,12 @@ package uk.ac.ebi.eva.accession.dbsnp2.runner; import com.fasterxml.jackson.databind.JsonNode; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.batch.core.JobInstance; import org.springframework.batch.core.StepExecution; import org.springframework.batch.core.explore.JobExplorer; @@ -33,15 +33,16 @@ import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; - +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.eva.accession.core.runner.CommandLineRunnerUtils; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.dbsnp2.batch.io.BzipLazyResource; import uk.ac.ebi.eva.accession.dbsnp2.parameters.InputParameters; +import uk.ac.ebi.eva.accession.dbsnp2.test.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.accession.dbsnp2.test.BatchTestConfiguration; +import uk.ac.ebi.eva.accession.dbsnp2.test.MongoTestConfiguration; import uk.ac.ebi.eva.commons.core.utils.FileUtils; -import javax.sql.DataSource; import java.io.BufferedReader; import java.io.File; import java.io.IOException; @@ -52,18 +53,19 @@ import java.util.Arrays; import java.util.Collections; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; import static uk.ac.ebi.eva.accession.dbsnp2.configuration.BeanNames.IMPORT_DBSNP_JSON_VARIANTS_JOB; import static uk.ac.ebi.eva.accession.dbsnp2.configuration.BeanNames.IMPORT_DBSNP_JSON_VARIANTS_STEP; import static uk.ac.ebi.eva.accession.dbsnp2.runner.DbsnpJsonImportVariantsJobLauncherCommandLineRunner.EXIT_WITHOUT_ERRORS; import static uk.ac.ebi.eva.accession.dbsnp2.runner.DbsnpJsonImportVariantsJobLauncherCommandLineRunner.EXIT_WITH_ERRORS; -@RunWith(SpringRunner.class) -@ContextConfiguration(classes = {BatchTestConfiguration.class}) +@ExtendWith(SpringExtension.class) +@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class, + BatchJobRepositoryTestConfiguration.class}) @TestPropertySource("classpath:application.properties") @SpringBatchTest -public class DbsnpJsonImportVariantsJobLauncherCommandLineRunnerTest { +public class DbsnpJsonImportVariantsJobLauncherCommandLineRunnerTest extends MongoTestContainerHelper { @Autowired private InputParameters inputParameters; @@ -74,9 +76,6 @@ public class DbsnpJsonImportVariantsJobLauncherCommandLineRunnerTest { @Autowired private JobExplorer jobExplorer; - @Autowired - private DataSource datasource; - @Autowired private DbsnpJsonImportVariantsJobLauncherCommandLineRunner runner; @@ -93,17 +92,17 @@ public class DbsnpJsonImportVariantsJobLauncherCommandLineRunnerTest { private boolean originalInputParametersCaptured = false; - @BeforeClass + @BeforeAll public static void initializeTempFile() throws Exception { tempJsonInputFileToTestFailingJobs = File.createTempFile("resumeFailingJob", ".json.bz2"); } - @AfterClass - public static void deleteTempFile() throws Exception { + @AfterAll + public static void deleteTempFile() { tempJsonInputFileToTestFailingJobs.delete(); } - @Before + @BeforeEach public void setUp() throws Exception { if (!originalInputParametersCaptured) { originalJsonInputFilePath = inputParameters.getInput(); @@ -111,14 +110,14 @@ public void setUp() throws Exception { writeToTempJsonFile(originalJsonContent); originalInputParametersCaptured = true; } - jobRepositoryTestUtils = new JobRepositoryTestUtils(jobRepository, datasource); - runner.setJobNames(IMPORT_DBSNP_JSON_VARIANTS_JOB); + jobRepositoryTestUtils = new JobRepositoryTestUtils(jobRepository); + runner.setJobName(IMPORT_DBSNP_JSON_VARIANTS_JOB); jobRepositoryTestUtils.removeJobExecutions(); inputParameters.setForceRestart(false); useOriginalJsonFile(); } - @After + @AfterEach public void tearDown() { jobRepositoryTestUtils.removeJobExecutions(); inputParameters.setForceRestart(false); @@ -159,11 +158,11 @@ private JobInstance runJobAandCheckResults() throws Exception { runner.run(); assertEquals(EXIT_WITH_ERRORS, runner.getExitCode()); JobInstance currentJobInstance = CommandLineRunnerUtils.getLastJobExecution(IMPORT_DBSNP_JSON_VARIANTS_JOB, - jobExplorer, - inputParameters.toJobParameters()) - .getJobInstance(); + jobExplorer, + inputParameters.toJobParameters()) + .getJobInstance(); StepExecution stepExecution = jobRepository.getLastStepExecution(currentJobInstance, - IMPORT_DBSNP_JSON_VARIANTS_STEP); + IMPORT_DBSNP_JSON_VARIANTS_STEP); //Ensure that only the first batch was written (batch size is 2 and error was at line#4) assertEquals(inputParameters.getChunkSize(), stepExecution.getWriteCount()); @@ -174,9 +173,9 @@ private void runJobBAndCheckRestart(JobInstance failingJobInstance) throws Excep runner.run(); assertEquals(EXIT_WITHOUT_ERRORS, runner.getExitCode()); JobInstance currentJobInstance = CommandLineRunnerUtils.getLastJobExecution(IMPORT_DBSNP_JSON_VARIANTS_JOB, - jobExplorer, - inputParameters.toJobParameters()) - .getJobInstance(); + jobExplorer, + inputParameters.toJobParameters()) + .getJobInstance(); assertNotEquals(failingJobInstance.getInstanceId(), currentJobInstance.getInstanceId()); } @@ -222,11 +221,11 @@ private void runJobBAndCheckResults() throws Exception { private void runJobCAndCheckResumption(JobInstance failingJobInstance) throws Exception { runner.run(); JobInstance currentJobInstance = CommandLineRunnerUtils.getLastJobExecution(IMPORT_DBSNP_JSON_VARIANTS_JOB, - jobExplorer, - inputParameters.toJobParameters()) - .getJobInstance(); + jobExplorer, + inputParameters.toJobParameters()) + .getJobInstance(); StepExecution stepExecution = jobRepository.getLastStepExecution(currentJobInstance, - IMPORT_DBSNP_JSON_VARIANTS_STEP); + IMPORT_DBSNP_JSON_VARIANTS_STEP); // Did we resume the previous failed job instance? assertEquals(failingJobInstance.getInstanceId(), currentJobInstance.getInstanceId()); @@ -294,7 +293,7 @@ private String getOriginalJsonContent(String inputJsonPath) throws IOException { private int getNumberOfLinesInJsonString(String jsonString) { return (int) Arrays.stream(jsonString.split(System.lineSeparator())) - .filter(line -> !line.startsWith("#")) - .count(); + .filter(line -> !line.startsWith("#")) + .count(); } } \ No newline at end of file diff --git a/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/test/BatchJobRepositoryTestConfiguration.java b/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/test/BatchJobRepositoryTestConfiguration.java new file mode 100644 index 000000000..dd4a09fb2 --- /dev/null +++ b/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/test/BatchJobRepositoryTestConfiguration.java @@ -0,0 +1,22 @@ +package uk.ac.ebi.eva.accession.dbsnp2.test; + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Primary; +import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseBuilder; +import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseType; + +import javax.sql.DataSource; + +@Configuration +public class BatchJobRepositoryTestConfiguration { + + @Bean + @Primary + public DataSource dataSource() { + return new EmbeddedDatabaseBuilder() + .setType(EmbeddedDatabaseType.HSQL) + .addScript("org/springframework/batch/core/schema-hsqldb.sql") + .build(); + } +} \ No newline at end of file diff --git a/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/test/BatchTestConfiguration.java b/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/test/BatchTestConfiguration.java index ffa8b1a8c..e88926287 100644 --- a/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/test/BatchTestConfiguration.java +++ b/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/test/BatchTestConfiguration.java @@ -15,29 +15,31 @@ */ package uk.ac.ebi.eva.accession.dbsnp2.test; +import org.springframework.batch.core.Job; import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; +import org.springframework.batch.core.launch.JobLauncher; +import org.springframework.batch.core.repository.JobRepository; import org.springframework.batch.test.JobLauncherTestUtils; -import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; -import org.springframework.boot.autoconfigure.batch.BatchProperties; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Import; -import org.springframework.transaction.PlatformTransactionManager; - import uk.ac.ebi.eva.accession.core.configuration.nonhuman.MongoConfiguration; -import uk.ac.ebi.eva.accession.dbsnp2.configuration.batch.policies.ChunkSizeCompletionPolicyConfiguration; +import uk.ac.ebi.eva.accession.dbsnp2.configuration.InputParametersConfiguration; import uk.ac.ebi.eva.accession.dbsnp2.configuration.batch.flow.ImportDbsnpJsonFlowConfiguration; -import uk.ac.ebi.eva.accession.dbsnp2.configuration.batch.jobs.ImportDbsnpJsonVariantsJobConfiguration; import uk.ac.ebi.eva.accession.dbsnp2.configuration.batch.io.ImportDbsnpJsonVariantsReaderConfiguration; -import uk.ac.ebi.eva.accession.dbsnp2.configuration.batch.steps.ImportDbsnpJsonVariantsStepConfiguration; import uk.ac.ebi.eva.accession.dbsnp2.configuration.batch.io.ImportDbsnpJsonVariantsWriterConfiguration; -import uk.ac.ebi.eva.accession.dbsnp2.configuration.InputParametersConfiguration; -import uk.ac.ebi.eva.accession.dbsnp2.configuration.batch.processors.JsonNodeToClusteredVariantProcessorConfiguration; +import uk.ac.ebi.eva.accession.dbsnp2.configuration.batch.jobs.ImportDbsnpJsonVariantsJobConfiguration; import uk.ac.ebi.eva.accession.dbsnp2.configuration.batch.listeners.ListenersConfiguration; +import uk.ac.ebi.eva.accession.dbsnp2.configuration.batch.policies.ChunkSizeCompletionPolicyConfiguration; +import uk.ac.ebi.eva.accession.dbsnp2.configuration.batch.processors.JsonNodeToClusteredVariantProcessorConfiguration; +import uk.ac.ebi.eva.accession.dbsnp2.configuration.batch.steps.ImportDbsnpJsonVariantsStepConfiguration; import uk.ac.ebi.eva.accession.dbsnp2.runner.DbsnpJsonImportVariantsJobLauncherCommandLineRunner; import uk.ac.ebi.eva.commons.batch.job.JobExecutionApplicationListener; +import static uk.ac.ebi.eva.accession.dbsnp2.configuration.BeanNames.IMPORT_DBSNP_JSON_VARIANTS_JOB; + @Configuration @EnableAutoConfiguration @EnableBatchProcessing @@ -54,15 +56,16 @@ DbsnpJsonImportVariantsJobLauncherCommandLineRunner.class}) public class BatchTestConfiguration { - @Autowired - private BatchProperties properties; - - @Autowired - private PlatformTransactionManager platformTransactionManager; + public static final String JOB_IMPORT_DBSNP_JSON_VARIANTS_JOB = "JOB_IMPORT_DBSNP_JSON_VARIANTS_JOB"; - @Bean - public JobLauncherTestUtils jobLauncherTestUtils() { - return new JobLauncherTestUtils(); + @Bean(JOB_IMPORT_DBSNP_JSON_VARIANTS_JOB) + public JobLauncherTestUtils jobLauncherTestUtilsFromMongo(JobLauncher jobLauncher, JobRepository jobRepository, + @Qualifier(IMPORT_DBSNP_JSON_VARIANTS_JOB) Job job) { + JobLauncherTestUtils utils = new JobLauncherTestUtils(); + utils.setJobLauncher(jobLauncher); + utils.setJobRepository(jobRepository); + utils.setJob(job); + return utils; } @Bean diff --git a/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/test/MongoTestConfiguration.java b/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/test/MongoTestConfiguration.java new file mode 100644 index 000000000..63280b6dd --- /dev/null +++ b/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/test/MongoTestConfiguration.java @@ -0,0 +1,102 @@ +/* + * Copyright 2021 EMBL - European Bioinformatics Institute + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package uk.ac.ebi.eva.accession.dbsnp2.test; + +import com.mongodb.ReadPreference; +import com.mongodb.WriteConcern; +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoClients; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.domain.EntityScan; +import org.springframework.boot.test.autoconfigure.data.mongo.AutoConfigureDataMongo; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Primary; +import org.springframework.data.mongodb.MongoDatabaseFactory; +import org.springframework.data.mongodb.config.EnableMongoAuditing; +import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.data.mongodb.core.SimpleMongoClientDatabaseFactory; +import org.springframework.data.mongodb.core.WriteResultChecking; +import org.springframework.data.mongodb.core.convert.DefaultDbRefResolver; +import org.springframework.data.mongodb.core.convert.DefaultMongoTypeMapper; +import org.springframework.data.mongodb.core.convert.MappingMongoConverter; +import org.springframework.data.mongodb.core.convert.MongoCustomConversions; +import org.springframework.data.mongodb.core.mapping.MongoMappingContext; +import org.springframework.data.mongodb.repository.config.EnableMongoRepositories; + +import java.util.Collections; + +@Configuration +@EntityScan(basePackages = {"uk.ac.ebi.eva.accession.core.persistence"}) +@EnableMongoRepositories(basePackages = { + "uk.ac.ebi.eva.accession.core.persistence", + "uk.ac.ebi.ampt2d.commons.accession.persistence.mongodb.repository"}) +@EnableMongoAuditing +@AutoConfigureDataMongo +public class MongoTestConfiguration { + + @Value("${spring.data.mongodb.uri}") + private String mongoUri; + + @Value("${spring.data.mongodb.database}") + private String database; + + @Value("${mongodb.read-preference:primary}") + private String readPreference; + + @Bean + @Primary + public MongoClient mongoClient() { + return MongoClients.create(mongoUri); + } + + @Bean("primaryFactory") + @Primary + public MongoDatabaseFactory mongoDbFactory() { + return new SimpleMongoClientDatabaseFactory(mongoClient(), database); + } + + @Bean + @Primary + public MongoCustomConversions mongoCustomConversions() { + return new MongoCustomConversions(Collections.emptyList()); + } + + @Bean + @Primary + public MongoMappingContext mongoMappingContext(MongoCustomConversions conversions) { + MongoMappingContext context = new MongoMappingContext(); + context.setSimpleTypeHolder(conversions.getSimpleTypeHolder()); + context.setAutoIndexCreation(false); + return context; + } + + @Bean + @Primary + public MongoTemplate mongoTemplate(MongoMappingContext mongoMappingContext, + MongoCustomConversions conversions) { + MappingMongoConverter converter = new MappingMongoConverter( + new DefaultDbRefResolver(mongoDbFactory()), mongoMappingContext); + converter.setCustomConversions(conversions); + converter.setTypeMapper(new DefaultMongoTypeMapper(null)); + converter.afterPropertiesSet(); + MongoTemplate mongoTemplate = new MongoTemplate(mongoDbFactory(), converter); + mongoTemplate.setReadPreference(ReadPreference.valueOf(readPreference)); + mongoTemplate.setWriteConcern(WriteConcern.MAJORITY); + mongoTemplate.setWriteResultChecking(WriteResultChecking.EXCEPTION); + return mongoTemplate; + } +} diff --git a/eva-accession-import-dbsnp2/src/test/resources/application.properties b/eva-accession-import-dbsnp2/src/test/resources/application.properties index 492409f84..d2e66060f 100644 --- a/eva-accession-import-dbsnp2/src/test/resources/application.properties +++ b/eva-accession-import-dbsnp2/src/test/resources/application.properties @@ -13,10 +13,7 @@ parameters.chunkSize=5 parameters.forceRestart=false # MongoDB for storing imported accessions -spring.data.mongodb.host=localhost -spring.data.mongodb.port=27017 spring.data.mongodb.database=admin -spring.data.mongodb.password= mongodb.read-preference=primaryPreferred diff --git a/eva-accession-import-dbsnp2/src/test/resources/test-dbsnp-json-import.properties b/eva-accession-import-dbsnp2/src/test/resources/test-dbsnp-json-import.properties deleted file mode 100644 index e69de29bb..000000000 diff --git a/eva-accession-pipeline/pom.xml b/eva-accession-pipeline/pom.xml index 21bd4ead9..34e3f8e38 100644 --- a/eva-accession-pipeline/pom.xml +++ b/eva-accession-pipeline/pom.xml @@ -7,7 +7,7 @@ eva-accession uk.ac.ebi.eva - 0.6.57-SNAPSHOT + ${revision} eva-accession-pipeline @@ -32,14 +32,37 @@ postgresql + + com.google.code.gson + gson + + + + uk.ac.ebi.eva + eva-accession-core + ${revision} + test-jar + test + + org.springframework.batch spring-batch-test test - com.lordofthejars - nosqlunit-mongodb + org.springframework.boot + spring-boot-testcontainers + test + + + org.testcontainers + mongodb + test + + + org.testcontainers + junit-jupiter test @@ -91,8 +114,4 @@ - - localhost - - diff --git a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/batch/io/AccessionReportWriter.java b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/batch/io/AccessionReportWriter.java index e209dfb0d..d1e5d867e 100644 --- a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/batch/io/AccessionReportWriter.java +++ b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/batch/io/AccessionReportWriter.java @@ -22,13 +22,12 @@ import org.springframework.batch.item.ItemStreamException; import uk.ac.ebi.ampt2d.commons.accession.core.models.AccessionWrapper; import uk.ac.ebi.ampt2d.commons.accession.core.models.GetOrCreateAccessionWrapper; - -import uk.ac.ebi.eva.accession.core.model.ISubmittedVariant; -import uk.ac.ebi.eva.accession.core.model.SubmittedVariant; +import uk.ac.ebi.eva.accession.core.batch.io.FastaSequenceReader; import uk.ac.ebi.eva.accession.core.contig.ContigMapping; import uk.ac.ebi.eva.accession.core.contig.ContigNaming; import uk.ac.ebi.eva.accession.core.contig.ContigSynonyms; -import uk.ac.ebi.eva.accession.core.batch.io.FastaSequenceReader; +import uk.ac.ebi.eva.accession.core.model.ISubmittedVariant; +import uk.ac.ebi.eva.accession.core.model.SubmittedVariant; import uk.ac.ebi.eva.accession.pipeline.batch.processors.ContigToGenbankReplacerProcessor; import uk.ac.ebi.eva.accession.pipeline.batch.tasklets.reportCheck.AccessionWrapperComparator; import uk.ac.ebi.eva.commons.core.models.IVariant; @@ -114,6 +113,7 @@ public void setAccessionPrefix(String accessionPrefix) { /** * We do not load the variants written in previous failed executions, but there might be duplicates hard to avoid, + * * @see AccessionReportWriterTest#resumeWritingWithRepeatedVariant */ public void open(ExecutionContext executionContext) throws ItemStreamException { @@ -132,16 +132,16 @@ public void open(ExecutionContext executionContext) throws ItemStreamException { } else { throw new IllegalStateException( "Can not resume step safely. All temporary files from the previous execution (" - + contigsOutput.getAbsolutePath() + " and " + variantsOutput.getAbsolutePath() - + ") should exist to resume the step. Please delete those files and start a new job."); + + contigsOutput.getAbsolutePath() + " and " + variantsOutput.getAbsolutePath() + + ") should exist to resume the step. Please delete those files and start a new job."); } } else { // we started a new job if (contigsOutput.exists() || variantsOutput.exists()) { throw new IllegalStateException( "A new job was started (did not resume a previous job) but temporary files exist (" - + contigsOutput.getAbsolutePath() + " or " + variantsOutput.getAbsolutePath() - + "). Please delete them and start a new job"); + + contigsOutput.getAbsolutePath() + " or " + variantsOutput.getAbsolutePath() + + "). Please delete them and start a new job"); } else { boolean append = false; this.contigsWriter = new BufferedWriter(new FileWriter(this.contigsOutput, append)); @@ -165,8 +165,8 @@ private void loadContigMappingFromTemporaryFile(File contigMappingFile) throws I String[] contigColumns = line.split("\t"); if (contigColumns.length != 2) { throw new IllegalStateException("Temporary file " + contigMappingFile.getAbsolutePath() - + " doesn't have the expected format. Please delete it and " - + "start a new job."); + + " doesn't have the expected format. Please delete it and " + + "start a new job."); } String inputContig = contigColumns[0]; String insdcContigAccession = contigColumns[1]; @@ -186,16 +186,16 @@ public void close() throws ItemStreamException { if (!duplicatedInputContigsToInsdc.isEmpty()) { logger.error( "The same chromosome (in the original input) was replaced by several INSDC contig accessions. " - + "This happened for: " + duplicatedInputContigsToInsdc.toString()); + + "This happened for: " + duplicatedInputContigsToInsdc.toString()); } if (!duplicatedInsdcToInputContigs.isEmpty()) { logger.error("The same INSDC contig accessions replaced several input chromosomes. This happened for: " - + duplicatedInsdcToInputContigs.toString()); + + duplicatedInsdcToInputContigs.toString()); } if (!duplicatedInputContigsToInsdc.isEmpty() || !duplicatedInsdcToInputContigs.isEmpty()) { throw new IllegalStateException( "Contig replacement was done but the replacements were not unique. See errors above for " - + "details"); + + "details"); } } catch (IOException e) { throw new ItemStreamException(e); @@ -206,7 +206,7 @@ public void write(List originalVariantsWithInsdcContigs, List> accessionedVariants) throws IOException { if (variantsWriter == null) { throw new IOException("The file " + variantsOutput + " was not opened properly. Hint: Check that the code " - + "called " + this.getClass().getSimpleName() + "::open"); + + "called " + this.getClass().getSimpleName() + "::open"); } updateChromosomeMappings(originalVariantsWithInsdcContigs); List> denormalizedVariants = denormalizeVariants( @@ -225,7 +225,7 @@ private void updateChromosomeMappings(List originalVariantsW String currentInsdcContig = variantWithContig.getChromosome(); String previousInsdcReplacementForOriginalChromosome = inputContigsToInsdc.put(originalChromosome, - currentInsdcContig); + currentInsdcContig); if (previousInsdcReplacementForOriginalChromosome == null) { // there was no previous entry, so this is not present in the contigs file: write it contigsWriter.write(originalChromosome + "\t" + currentInsdcContig); @@ -250,17 +250,17 @@ private void updateChromosomeMappings(List originalVariantsW private String getOriginalChromosome(IVariant variant) { Set originalChromosomes = variant.getSourceEntries() - .stream() - .map(se -> se.getAttributes().get( - ContigToGenbankReplacerProcessor.ORIGINAL_CHROMOSOME)) - .collect(Collectors.toSet()); + .stream() + .map(se -> se.getAttributes().get( + ContigToGenbankReplacerProcessor.ORIGINAL_CHROMOSOME)) + .collect(Collectors.toSet()); if (originalChromosomes.size() != 1) { throw new IllegalStateException( "Bug detected: Multiple original chromosomes were found to be associated with the same variant " - + variant.toString() + ". The attributes had the next list of original chromosomes: [" - + String.join(", ", originalChromosomes) + "]. Contig '" + variant.getChromosome() - + "' was used as replacement."); + + variant.toString() + ". The attributes had the next list of original chromosomes: [" + + String.join(", ", originalChromosomes) + "]. Contig '" + variant.getChromosome() + + "' was used as replacement."); } return originalChromosomes.iterator().next(); } @@ -270,7 +270,7 @@ private List> denorm List> denormalizedAccessions = new ArrayList<>(); for (AccessionWrapper accession : accessions) { denormalizedAccessions.add(new AccessionWrapper<>(accession.getAccession(), accession.getHash(), - denormalizeVariant(accession.getData()))); + denormalizeVariant(accession.getData()))); } return denormalizedAccessions; } @@ -281,7 +281,7 @@ private ISubmittedVariant denormalizeVariant(ISubmittedVariant normalizedVariant return createVariantWithContextBase(normalizedVariant); } else { throw new IllegalArgumentException("Contig '" + normalizedVariant.getContig() - + "' does not appear in the FASTA file "); + + "' does not appear in the FASTA file "); } } else { return normalizedVariant; @@ -294,30 +294,30 @@ private ISubmittedVariant createVariantWithContextBase(ISubmittedVariant normali long oldStart = normalizedVariant.getStart(); ImmutableTriple contextNucleotideInfo = fastaSequenceReader.getContextNucleotideAndNewStart(normalizedVariant.getContig(), oldStart, - oldReference, oldAlternate); + oldReference, oldAlternate); return new SubmittedVariant(normalizedVariant.getReferenceSequenceAccession(), - normalizedVariant.getTaxonomyAccession(), - normalizedVariant.getProjectAccession(), - normalizedVariant.getContig(), - contextNucleotideInfo.getLeft(), - contextNucleotideInfo.getMiddle(), - contextNucleotideInfo.getRight(), - normalizedVariant.getClusteredVariantAccession(), - normalizedVariant.isSupportedByEvidence(), - normalizedVariant.isAssemblyMatch(), - normalizedVariant.isAllelesMatch(), - normalizedVariant.isValidated(), - normalizedVariant.getCreatedDate()); + normalizedVariant.getTaxonomyAccession(), + normalizedVariant.getProjectAccession(), + normalizedVariant.getContig(), + contextNucleotideInfo.getLeft(), + contextNucleotideInfo.getMiddle(), + contextNucleotideInfo.getRight(), + normalizedVariant.getClusteredVariantAccession(), + normalizedVariant.isSupportedByEvidence(), + normalizedVariant.isAssemblyMatch(), + normalizedVariant.isAllelesMatch(), + normalizedVariant.isValidated(), + normalizedVariant.getCreatedDate()); } /** * Replace the contig using the requested contig naming and write the variant to the output file. - * + *

* Note how this is done after the sorting (using {@link AccessionWrapperComparator}) because the mappings we * passed to it are mappings from the input naming to INSDC, not from input naming to requested output naming. - * + *

* Also, we have to get the equivalent of the original chromosome (not the INSDC contig) because we will mostly use * {@link ContigNaming#NO_REPLACEMENT} and it means "do not replace the submitter's original chromosome". */ @@ -327,12 +327,12 @@ private void writeSortedVariant(AccessionWrapper variants) throws Exception { + public void write(Chunk variants) throws Exception { if (!variants.isEmpty()) { - List submittedVariants = variants.stream().map(variantConverter::convert) + List submittedVariants = variants.getItems().stream().map(variantConverter::convert) .collect(Collectors.toList()); List> accessions = service.getOrCreate(submittedVariants, jobExecution.getJobId().toString()); metricCompute.addCount(AccessioningMetric.SUBMITTED_VARIANTS, variants.size()); metricCompute.addCount(AccessioningMetric.ACCESSIONED_VARIANTS, accessions.size()); - accessionReportWriter.write(variants, accessions); + accessionReportWriter.write(variants.getItems(), accessions); checkCountsMatch(submittedVariants, accessions); } } @@ -77,28 +78,28 @@ void checkCountsMatch(List variants, List> accessions) { if (variants.size() != accessions.size()) { Set accessionedVariants = accessions.stream() - .map(AccessionWrapper::getData) - .map(this::getSubmittedVariantWithoutClusteredVariantAccession) - .collect(Collectors.toSet()); + .map(AccessionWrapper::getData) + .map(this::getSubmittedVariantWithoutClusteredVariantAccession) + .collect(Collectors.toSet()); HashSet distinctVariants = new HashSet<>(variants); int duplicateCount = variants.size() - distinctVariants.size(); metricCompute.addCount(AccessioningMetric.DISTINCT_VARIANTS, distinctVariants.size()); metricCompute.addCount(AccessioningMetric.DUPLICATE_VARIANTS, duplicateCount); if (duplicateCount != 0) { logger.warn("A variant chunk contains {} repeated variants. This is not an error, but please check " + - "it's expected.", duplicateCount); + "it's expected.", duplicateCount); } Set variantsWithoutAccession = distinctVariants.stream() - .filter(v -> !accessionedVariants - .contains(v)) - .collect(Collectors.toSet()); + .filter(v -> !accessionedVariants + .contains(v)) + .collect(Collectors.toSet()); metricCompute.addCount(AccessioningMetric.DISCARDED_VARIANTS, variantsWithoutAccession.size()); if (variantsWithoutAccession.size() != 0) { logger.error("A problem occurred while accessioning a chunk. Total num variants = {}, distinct = {}, " + - "duplicate = {}, accessioned = {}, not accessioned = {}", - variants.size(), distinctVariants.size(), duplicateCount, accessionedVariants.size(), - variantsWithoutAccession.size()); + "duplicate = {}, accessioned = {}, not accessioned = {}", + variants.size(), distinctVariants.size(), duplicateCount, accessionedVariants.size(), + variantsWithoutAccession.size()); logger.error("The non-accessioned variants are: {}", variantsWithoutAccession.toString()); throw new IllegalStateException( @@ -109,18 +110,18 @@ void checkCountsMatch(List variants, private ISubmittedVariant getSubmittedVariantWithoutClusteredVariantAccession(ISubmittedVariant submittedVariant) { return new SubmittedVariant(submittedVariant.getReferenceSequenceAccession(), - submittedVariant.getTaxonomyAccession(), - submittedVariant.getProjectAccession(), - submittedVariant.getContig(), - submittedVariant.getStart(), - submittedVariant.getReferenceAllele(), - submittedVariant.getAlternateAllele(), - null, - submittedVariant.isSupportedByEvidence(), - submittedVariant.isAssemblyMatch(), - submittedVariant.isAllelesMatch(), - submittedVariant.isValidated(), - submittedVariant.getCreatedDate()); + submittedVariant.getTaxonomyAccession(), + submittedVariant.getProjectAccession(), + submittedVariant.getContig(), + submittedVariant.getStart(), + submittedVariant.getReferenceAllele(), + submittedVariant.getAlternateAllele(), + null, + submittedVariant.isSupportedByEvidence(), + submittedVariant.isAssemblyMatch(), + submittedVariant.isAllelesMatch(), + submittedVariant.isValidated(), + submittedVariant.getCreatedDate()); } @Override diff --git a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/batch/io/DuplicateSSAccQCWriter.java b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/batch/io/DuplicateSSAccQCWriter.java index b9a26a4bb..84defd8dd 100644 --- a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/batch/io/DuplicateSSAccQCWriter.java +++ b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/batch/io/DuplicateSSAccQCWriter.java @@ -1,9 +1,10 @@ package uk.ac.ebi.eva.accession.pipeline.batch.io; -import gherkin.deps.com.google.gson.Gson; -import gherkin.deps.com.google.gson.GsonBuilder; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.batch.item.Chunk; import org.springframework.batch.item.ExecutionContext; import org.springframework.batch.item.ItemStreamException; import org.springframework.batch.item.ItemStreamWriter; @@ -48,7 +49,7 @@ public void close() throws ItemStreamException { } @Override - public void write(List> listOfDuplicateSSAccQCResultLists) throws Exception { + public void write(Chunk> listOfDuplicateSSAccQCResultLists) throws Exception { for (List duplicateSSAccQCResultList : listOfDuplicateSSAccQCResultLists) { if (duplicateSSAccQCResultList != null && !duplicateSSAccQCResultList.isEmpty()) { appendToFile(duplicateSSAccQCResultList); diff --git a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/batch/policies/InvalidVariantSkipPolicy.java b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/batch/policies/InvalidVariantSkipPolicy.java index f6b6c5c3b..cb5e4984d 100644 --- a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/batch/policies/InvalidVariantSkipPolicy.java +++ b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/batch/policies/InvalidVariantSkipPolicy.java @@ -31,7 +31,7 @@ public class InvalidVariantSkipPolicy implements SkipPolicy { @Override - public boolean shouldSkip(Throwable exception, int skipCount) throws SkipLimitExceededException { + public boolean shouldSkip(Throwable exception, long skipCount) throws SkipLimitExceededException { if (exception instanceof FlatFileParseException && (exception.getCause() instanceof NonVariantException || exception.getCause() instanceof IncompleteInformationException)) { diff --git a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/RunnerConfiguration.java b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/RunnerConfiguration.java index 517525c34..6b60049ff 100644 --- a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/RunnerConfiguration.java +++ b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/RunnerConfiguration.java @@ -16,16 +16,10 @@ */ package uk.ac.ebi.eva.accession.pipeline.configuration; -import org.springframework.batch.core.configuration.annotation.BatchConfigurer; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; - -import uk.ac.ebi.eva.commons.batch.configuration.SpringBoot1CompatibilityConfiguration; import uk.ac.ebi.eva.commons.batch.job.JobExecutionApplicationListener; -import javax.persistence.EntityManagerFactory; -import javax.sql.DataSource; - @Configuration public class RunnerConfiguration { @@ -33,11 +27,4 @@ public class RunnerConfiguration { public JobExecutionApplicationListener jobExecutionApplicationListener() { return new JobExecutionApplicationListener(); } - - @Bean - public BatchConfigurer configurer(DataSource dataSource, EntityManagerFactory entityManagerFactory) - throws Exception { - return SpringBoot1CompatibilityConfiguration.getSpringBoot1CompatibleBatchConfigurer(dataSource, - entityManagerFactory); - } } diff --git a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/io/AccessionWriterConfiguration.java b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/io/AccessionWriterConfiguration.java index 34baa7fe7..8071acec2 100644 --- a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/io/AccessionWriterConfiguration.java +++ b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/io/AccessionWriterConfiguration.java @@ -18,17 +18,16 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Import; - -import uk.ac.ebi.eva.accession.core.service.nonhuman.SubmittedVariantAccessioningService; +import uk.ac.ebi.eva.accession.core.batch.io.FastaSynonymSequenceReader; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.SubmittedVariantAccessioningConfiguration; import uk.ac.ebi.eva.accession.core.contig.ContigMapping; -import uk.ac.ebi.eva.accession.core.batch.io.FastaSynonymSequenceReader; -import uk.ac.ebi.eva.accession.pipeline.configuration.InputParametersConfiguration; +import uk.ac.ebi.eva.accession.core.service.nonhuman.SubmittedVariantAccessioningService; import uk.ac.ebi.eva.accession.pipeline.batch.io.AccessionReportWriter; import uk.ac.ebi.eva.accession.pipeline.batch.io.AccessionWriter; -import uk.ac.ebi.eva.accession.pipeline.parameters.InputParameters; import uk.ac.ebi.eva.accession.pipeline.batch.processors.VariantConverter; +import uk.ac.ebi.eva.accession.pipeline.configuration.InputParametersConfiguration; import uk.ac.ebi.eva.accession.pipeline.configuration.batch.listeners.ListenersConfiguration; +import uk.ac.ebi.eva.accession.pipeline.parameters.InputParameters; import uk.ac.ebi.eva.metrics.metric.MetricCompute; import java.io.File; diff --git a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/io/VcfReaderConfiguration.java b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/io/VcfReaderConfiguration.java index 2a8c75996..0123827b3 100644 --- a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/io/VcfReaderConfiguration.java +++ b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/io/VcfReaderConfiguration.java @@ -21,14 +21,13 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Import; - import uk.ac.ebi.eva.accession.pipeline.configuration.InputParametersConfiguration; import uk.ac.ebi.eva.accession.pipeline.parameters.InputParameters; +import uk.ac.ebi.eva.commons.batch.io.AggregatedVcfReader; import uk.ac.ebi.eva.commons.batch.io.UnwindingItemStreamReader; +import uk.ac.ebi.eva.commons.batch.io.VcfReader; import uk.ac.ebi.eva.commons.core.models.Aggregation; import uk.ac.ebi.eva.commons.core.models.pipeline.Variant; -import uk.ac.ebi.eva.commons.batch.io.AggregatedVcfReader; -import uk.ac.ebi.eva.commons.batch.io.VcfReader; import java.io.File; import java.io.IOException; diff --git a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/DuplicateSSAccQCJobConfiguration.java b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/DuplicateSSAccQCJobConfiguration.java index 7ee57a6c4..b3f570c39 100644 --- a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/DuplicateSSAccQCJobConfiguration.java +++ b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/DuplicateSSAccQCJobConfiguration.java @@ -3,8 +3,9 @@ import org.springframework.batch.core.Job; import org.springframework.batch.core.Step; import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; -import org.springframework.batch.core.configuration.annotation.JobBuilderFactory; +import org.springframework.batch.core.job.builder.JobBuilder; import org.springframework.batch.core.launch.support.RunIdIncrementer; +import org.springframework.batch.core.repository.JobRepository; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; @@ -22,8 +23,8 @@ public class DuplicateSSAccQCJobConfiguration { private Step duplicateSSAccQCStep; @Bean(DUPLICATE_SS_ACC_QC_JOB) - public Job duplicateSSAccQCJob(JobBuilderFactory jobBuilderFactory) { - return jobBuilderFactory.get(DUPLICATE_SS_ACC_QC_JOB) + public Job duplicateSSAccQCJob(JobRepository jobRepository) { + return new JobBuilder(DUPLICATE_SS_ACC_QC_JOB, jobRepository) .incrementer(new RunIdIncrementer()) .start(duplicateSSAccQCStep) .build(); diff --git a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/QCSubsnpAccessionsJobConfiguration.java b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/QCSubsnpAccessionsJobConfiguration.java index 5caafdd24..6ee75487b 100644 --- a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/QCSubsnpAccessionsJobConfiguration.java +++ b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/QCSubsnpAccessionsJobConfiguration.java @@ -3,8 +3,9 @@ import org.springframework.batch.core.Job; import org.springframework.batch.core.Step; import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; -import org.springframework.batch.core.configuration.annotation.JobBuilderFactory; +import org.springframework.batch.core.job.builder.JobBuilder; import org.springframework.batch.core.launch.support.RunIdIncrementer; +import org.springframework.batch.core.repository.JobRepository; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; @@ -21,8 +22,8 @@ public class QCSubsnpAccessionsJobConfiguration { private Step qcSubsnpAccessionStep; @Bean(QC_SUBSNP_ACCESSION_JOB) - public Job qcSubsnpAccessionJob(JobBuilderFactory jobBuilderFactory) { - return jobBuilderFactory.get(QC_SUBSNP_ACCESSION_JOB) + public Job qcSubsnpAccessionJob(JobRepository jobRepository) { + return new JobBuilder(QC_SUBSNP_ACCESSION_JOB, jobRepository) .incrementer(new RunIdIncrementer()) .start(qcSubsnpAccessionStep) .build(); diff --git a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/SSAccessionRecoveryJobConfiguration.java b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/SSAccessionRecoveryJobConfiguration.java index 25a2669d2..878ed8ad9 100644 --- a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/SSAccessionRecoveryJobConfiguration.java +++ b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/SSAccessionRecoveryJobConfiguration.java @@ -4,8 +4,9 @@ import org.springframework.batch.core.JobExecutionListener; import org.springframework.batch.core.Step; import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; -import org.springframework.batch.core.configuration.annotation.JobBuilderFactory; +import org.springframework.batch.core.job.builder.JobBuilder; import org.springframework.batch.core.launch.support.RunIdIncrementer; +import org.springframework.batch.core.repository.JobRepository; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; @@ -28,8 +29,8 @@ public class SSAccessionRecoveryJobConfiguration { private JobExecutionListener ssAccessionRecoveryJobListener; @Bean(SS_ACCESSION_RECOVERY_JOB) - public Job createSSAccessionRecoveryJob(JobBuilderFactory jobBuilderFactory) { - return jobBuilderFactory.get(SS_ACCESSION_RECOVERY_JOB) + public Job createSSAccessionRecoveryJob(JobRepository jobRepository) { + return new JobBuilder(SS_ACCESSION_RECOVERY_JOB, jobRepository) .incrementer(new RunIdIncrementer()) .start(ssAccessionRecoveryStep) .listener(ssAccessionRecoveryJobListener) diff --git a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/SubsnpAccessionsJobConfiguration.java b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/SubsnpAccessionsJobConfiguration.java index 9907ff169..43290594d 100644 --- a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/SubsnpAccessionsJobConfiguration.java +++ b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/SubsnpAccessionsJobConfiguration.java @@ -20,8 +20,9 @@ import org.springframework.batch.core.JobExecutionListener; import org.springframework.batch.core.Step; import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; -import org.springframework.batch.core.configuration.annotation.JobBuilderFactory; +import org.springframework.batch.core.job.builder.JobBuilder; import org.springframework.batch.core.launch.support.RunIdIncrementer; +import org.springframework.batch.core.repository.JobRepository; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; @@ -54,8 +55,8 @@ public class SubsnpAccessionsJobConfiguration { private JobExecutionListener subsnpAccessionJobListener; @Bean(SUBSNP_ACCESSION_JOB) - public Job subsnpAccessionJob(JobBuilderFactory jobBuilderFactory) { - return jobBuilderFactory.get(SUBSNP_ACCESSION_JOB) + public Job subsnpAccessionJob(JobRepository jobRepository) { + return new JobBuilder(SUBSNP_ACCESSION_JOB, jobRepository) .incrementer(new RunIdIncrementer()) .start(subsnpAccessionStep) .next(accessioningShutdownStep) diff --git a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/processors/VariantProcessorConfiguration.java b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/processors/VariantProcessorConfiguration.java index f164b54aa..56d476846 100644 --- a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/processors/VariantProcessorConfiguration.java +++ b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/processors/VariantProcessorConfiguration.java @@ -23,11 +23,10 @@ import org.springframework.batch.item.support.CompositeItemProcessor; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; - import uk.ac.ebi.eva.accession.core.contig.ContigMapping; -import uk.ac.ebi.eva.accession.pipeline.parameters.InputParameters; import uk.ac.ebi.eva.accession.pipeline.batch.processors.ContigToGenbankReplacerProcessor; import uk.ac.ebi.eva.accession.pipeline.batch.processors.ExcludeStructuralVariantsProcessor; +import uk.ac.ebi.eva.accession.pipeline.parameters.InputParameters; import uk.ac.ebi.eva.commons.core.models.IVariant; import java.util.Arrays; diff --git a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/AccessioningShutdownStepConfiguration.java b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/AccessioningShutdownStepConfiguration.java index 75f083f2d..ccad6721b 100644 --- a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/AccessioningShutdownStepConfiguration.java +++ b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/AccessioningShutdownStepConfiguration.java @@ -2,10 +2,12 @@ import org.springframework.batch.core.Step; import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; -import org.springframework.batch.core.configuration.annotation.StepBuilderFactory; +import org.springframework.batch.core.repository.JobRepository; +import org.springframework.batch.core.step.builder.StepBuilder; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.transaction.PlatformTransactionManager; import uk.ac.ebi.eva.accession.core.service.nonhuman.SubmittedVariantAccessioningService; import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.ACCESSIONING_SHUTDOWN_STEP; @@ -17,12 +19,12 @@ public class AccessioningShutdownStepConfiguration { private SubmittedVariantAccessioningService submittedVariantAccessioningService; @Bean(ACCESSIONING_SHUTDOWN_STEP) - public Step accessioningShutDownStep(StepBuilderFactory stepBuilderFactory) { - return stepBuilderFactory.get(ACCESSIONING_SHUTDOWN_STEP) + public Step accessioningShutDownStep(JobRepository jobRepository, PlatformTransactionManager transactionManager) { + return new StepBuilder(ACCESSIONING_SHUTDOWN_STEP, jobRepository) .tasklet((contribution, chunkContext) -> { submittedVariantAccessioningService.shutDownAccessionGenerator(); return null; - }) + }, transactionManager) .build(); } } \ No newline at end of file diff --git a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/BuildReportStepConfiguration.java b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/BuildReportStepConfiguration.java index 090273e2c..e7fe6d462 100644 --- a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/BuildReportStepConfiguration.java +++ b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/BuildReportStepConfiguration.java @@ -18,17 +18,17 @@ import org.springframework.batch.core.Step; import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; -import org.springframework.batch.core.configuration.annotation.StepBuilderFactory; +import org.springframework.batch.core.repository.JobRepository; +import org.springframework.batch.core.step.builder.StepBuilder; import org.springframework.batch.core.step.tasklet.TaskletStep; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; - -import uk.ac.ebi.eva.accession.pipeline.parameters.InputParameters; +import org.springframework.transaction.PlatformTransactionManager; import uk.ac.ebi.eva.accession.pipeline.batch.tasklets.buildReport.BuildReportTasklet; +import uk.ac.ebi.eva.accession.pipeline.parameters.InputParameters; import java.io.File; -import java.io.IOException; import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.BUILD_REPORT_STEP; @@ -40,11 +40,11 @@ public class BuildReportStepConfiguration { private InputParameters inputParameters; @Bean(BUILD_REPORT_STEP) - public Step buildReportStep(StepBuilderFactory stepBuilderFactory) throws IOException { + public Step buildReportStep(JobRepository jobRepository, PlatformTransactionManager transactionManager) { BuildReportTasklet tasklet = new BuildReportTasklet(new File(inputParameters.getOutputVcf())); - TaskletStep step = stepBuilderFactory.get(BUILD_REPORT_STEP) - .tasklet(tasklet) - .build(); + TaskletStep step = new StepBuilder(BUILD_REPORT_STEP, jobRepository) + .tasklet(tasklet, transactionManager) + .build(); return step; } } diff --git a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/DuplicateSSAccQCStepConfiguration.java b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/DuplicateSSAccQCStepConfiguration.java index 0364100a9..75a9cabde 100644 --- a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/DuplicateSSAccQCStepConfiguration.java +++ b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/DuplicateSSAccQCStepConfiguration.java @@ -2,7 +2,8 @@ import org.springframework.batch.core.Step; import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; -import org.springframework.batch.core.configuration.annotation.StepBuilderFactory; +import org.springframework.batch.core.repository.JobRepository; +import org.springframework.batch.core.step.builder.StepBuilder; import org.springframework.batch.core.step.tasklet.TaskletStep; import org.springframework.batch.item.ItemProcessor; import org.springframework.batch.item.ItemStreamReader; @@ -11,6 +12,7 @@ import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.transaction.PlatformTransactionManager; import uk.ac.ebi.eva.accession.pipeline.batch.io.DuplicateSSAccQCResult; import java.util.List; @@ -37,11 +39,11 @@ public class DuplicateSSAccQCStepConfiguration { private ItemWriter> duplicateSSAccQCWriter; @Bean(DUPLICATE_SS_ACC_QC_STEP) - public Step duplicateSSAccQCStep(StepBuilderFactory stepBuilderFactory) { - TaskletStep step = stepBuilderFactory.get(DUPLICATE_SS_ACC_QC_STEP) + public Step duplicateSSAccQCStep(JobRepository jobRepository, PlatformTransactionManager transactionManager) { + TaskletStep step = new StepBuilder(DUPLICATE_SS_ACC_QC_STEP, jobRepository) // hardcoded the chunk size as 1, as the reader takes care of accumulating // and sending the chunk size (defined in properties file) elements to the processor - ., List>chunk(1) + ., List>chunk(1, transactionManager) .reader(ssAccFileReader) .processor(duplicateSSAccQCProcessor) .writer(duplicateSSAccQCWriter) diff --git a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/QCSubsnpAccessionsStepConfiguration.java b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/QCSubsnpAccessionsStepConfiguration.java index 3af6a5eeb..6fe42f886 100644 --- a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/QCSubsnpAccessionsStepConfiguration.java +++ b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/QCSubsnpAccessionsStepConfiguration.java @@ -18,13 +18,15 @@ import org.springframework.batch.core.Step; import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; -import org.springframework.batch.core.configuration.annotation.StepBuilderFactory; +import org.springframework.batch.core.repository.JobRepository; +import org.springframework.batch.core.step.builder.StepBuilder; import org.springframework.batch.core.step.tasklet.TaskletStep; import org.springframework.batch.item.ItemStreamReader; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.transaction.PlatformTransactionManager; import uk.ac.ebi.eva.accession.core.batch.io.AccessionedVcfLineMapper; import uk.ac.ebi.eva.accession.core.contig.ContigMapping; import uk.ac.ebi.eva.accession.pipeline.batch.tasklets.reportCheck.ReportCheckTasklet; @@ -62,11 +64,11 @@ public ItemStreamReader reportReader() throws IOException { } @Bean(QC_SUBSNP_ACCESSION_STEP) - public Step qcSubsnpAccessionStep(StepBuilderFactory stepBuilderFactory) throws IOException { + public Step qcSubsnpAccessionStep(JobRepository jobRepository, PlatformTransactionManager transactionManager) throws IOException { ReportCheckTasklet tasklet = new ReportCheckTasklet(inputReader, reportReader(), inputParameters.getChunkSize() * 2, contigMapping); - TaskletStep step = stepBuilderFactory.get(QC_SUBSNP_ACCESSION_STEP) - .tasklet(tasklet) + TaskletStep step = new StepBuilder(QC_SUBSNP_ACCESSION_STEP, jobRepository) + .tasklet(tasklet, transactionManager) .build(); return step; } diff --git a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/SSAccessionRecoveryStepConfiguration.java b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/SSAccessionRecoveryStepConfiguration.java index 758d75616..a87432c36 100644 --- a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/SSAccessionRecoveryStepConfiguration.java +++ b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/SSAccessionRecoveryStepConfiguration.java @@ -2,11 +2,13 @@ import org.springframework.batch.core.Step; import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; -import org.springframework.batch.core.configuration.annotation.StepBuilderFactory; +import org.springframework.batch.core.repository.JobRepository; +import org.springframework.batch.core.step.builder.StepBuilder; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.transaction.PlatformTransactionManager; import uk.ac.ebi.eva.accession.pipeline.batch.recovery.SSAccessionRecoveryService; import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.SS_ACCESSION_RECOVERY_SERVICE; @@ -20,12 +22,12 @@ public class SSAccessionRecoveryStepConfiguration { private SSAccessionRecoveryService SSAccessionRecoveryService; @Bean(SS_ACCESSION_RECOVERY_STEP) - public Step ssAccessionRecoveryStep(StepBuilderFactory stepBuilderFactory) { - return stepBuilderFactory.get(SS_ACCESSION_RECOVERY_STEP) + public Step ssAccessionRecoveryStep(JobRepository jobRepository, PlatformTransactionManager transactionManager) { + return new StepBuilder(SS_ACCESSION_RECOVERY_STEP, jobRepository) .tasklet((contribution, chunkContext) -> { SSAccessionRecoveryService.runRecoveryForCategorySS(); return null; - }) + }, transactionManager) .build(); } } diff --git a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/SubsnpAccessionsStepConfiguration.java b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/SubsnpAccessionsStepConfiguration.java index 6ef6c30b4..8f4902761 100644 --- a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/SubsnpAccessionsStepConfiguration.java +++ b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/SubsnpAccessionsStepConfiguration.java @@ -19,7 +19,8 @@ import org.springframework.batch.core.Step; import org.springframework.batch.core.StepExecutionListener; import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; -import org.springframework.batch.core.configuration.annotation.StepBuilderFactory; +import org.springframework.batch.core.repository.JobRepository; +import org.springframework.batch.core.step.builder.StepBuilder; import org.springframework.batch.core.step.tasklet.TaskletStep; import org.springframework.batch.item.ItemProcessor; import org.springframework.batch.item.ItemReader; @@ -29,6 +30,7 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.retry.backoff.ExponentialBackOffPolicy; +import org.springframework.transaction.PlatformTransactionManager; import uk.ac.ebi.ampt2d.commons.accession.core.exceptions.MissingUnsavedAccessionsException; import uk.ac.ebi.eva.accession.pipeline.batch.io.AccessionWriter; import uk.ac.ebi.eva.accession.pipeline.batch.policies.InvalidVariantSkipPolicy; @@ -65,10 +67,10 @@ public class SubsnpAccessionsStepConfiguration { private InvalidVariantSkipPolicy invalidVariantSkipPolicy; @Bean(SUBSNP_ACCESSION_STEP) - public Step subsnpAccessionStep(StepBuilderFactory stepBuilderFactory, + public Step subsnpAccessionStep(JobRepository jobRepository, PlatformTransactionManager transactionManager, SimpleCompletionPolicy chunkSizeCompletionPolicy) { - TaskletStep step = stepBuilderFactory.get(SUBSNP_ACCESSION_STEP) - .chunk(chunkSizeCompletionPolicy) + TaskletStep step = new StepBuilder(SUBSNP_ACCESSION_STEP, jobRepository) + .chunk(chunkSizeCompletionPolicy, transactionManager) .reader(variantReader) .processor(variantProcessor) .writer(accessionWriter) diff --git a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/metric/AccessioningMetricCompute.java b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/metric/AccessioningMetricCompute.java index 16ec97902..4063888cc 100644 --- a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/metric/AccessioningMetricCompute.java +++ b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/metric/AccessioningMetricCompute.java @@ -1,7 +1,8 @@ package uk.ac.ebi.eva.accession.pipeline.metric; -import org.codehaus.jettison.json.JSONException; -import org.codehaus.jettison.json.JSONObject; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; import org.springframework.web.client.RestTemplate; import uk.ac.ebi.eva.metrics.count.CountServiceParameters; import uk.ac.ebi.eva.metrics.metric.BaseMetricCompute; @@ -11,6 +12,8 @@ import java.util.stream.Collectors; public class AccessioningMetricCompute extends BaseMetricCompute { + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + private static final String PROCESS = "accessioning_warehouse_ingestion"; private final String assembly; private final String study; @@ -32,12 +35,13 @@ public List getMetrics() { public String getIdentifier() { try { - JSONObject identifier = new JSONObject(); + ObjectNode identifier = OBJECT_MAPPER.createObjectNode(); identifier.put("assembly", assembly); identifier.put("study", study); - return identifier.toString(); - } catch (JSONException jsonException) { - throw new RuntimeException("Could not create Identifier for Accessioning Counts. Error ", jsonException); + + return OBJECT_MAPPER.writeValueAsString(identifier); + } catch (JsonProcessingException ex) { + throw new RuntimeException("Could not create Identifier for Accessioning Counts. Error ", ex); } } diff --git a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/runner/EvaAccessionJobLauncherCommandLineRunner.java b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/runner/EvaAccessionJobLauncherCommandLineRunner.java index aef346c20..adcc1202c 100644 --- a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/runner/EvaAccessionJobLauncherCommandLineRunner.java +++ b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/runner/EvaAccessionJobLauncherCommandLineRunner.java @@ -24,7 +24,6 @@ import org.springframework.batch.core.JobParametersInvalidException; import org.springframework.batch.core.explore.JobExplorer; import org.springframework.batch.core.launch.JobLauncher; -import org.springframework.batch.core.launch.JobParametersNotFoundException; import org.springframework.batch.core.repository.JobExecutionAlreadyRunningException; import org.springframework.batch.core.repository.JobInstanceAlreadyCompleteException; import org.springframework.batch.core.repository.JobRepository; @@ -32,11 +31,10 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.ExitCodeGenerator; -import org.springframework.boot.autoconfigure.batch.JobLauncherCommandLineRunner; +import org.springframework.boot.autoconfigure.batch.JobLauncherApplicationRunner; import org.springframework.context.ApplicationEventPublisherAware; import org.springframework.stereotype.Component; import org.springframework.util.PatternMatchUtils; - import uk.ac.ebi.eva.accession.core.runner.CommandLineRunnerUtils; import uk.ac.ebi.eva.accession.pipeline.parameters.InputParameters; import uk.ac.ebi.eva.commons.batch.exception.NoJobToExecuteException; @@ -56,7 +54,7 @@ * -The user can restart a job that has been run previously marking the previous execution as failed. */ @Component -public class EvaAccessionJobLauncherCommandLineRunner extends JobLauncherCommandLineRunner implements +public class EvaAccessionJobLauncherCommandLineRunner extends JobLauncherApplicationRunner implements ApplicationEventPublisherAware, ExitCodeGenerator { private static final Logger logger = LoggerFactory.getLogger(EvaAccessionJobLauncherCommandLineRunner.class); @@ -98,9 +96,9 @@ public void setJobs(Collection jobs) { } @Override - public void setJobNames(String jobName) { + public void setJobName(String jobName) { this.jobName = jobName; - super.setJobNames(jobName); + super.setJobName(jobName); } @Override @@ -124,16 +122,15 @@ public void run(String... args) throws JobExecutionException { JobStatusManager.checkIfPropertiesHaveBeenProvided(jobParameters); if (inputParameters.isForceRestart()) { JobExecution previousJobExecution = CommandLineRunnerUtils.getLastJobExecution(jobName, jobExplorer, - jobParameters); + jobParameters); CommandLineRunnerUtils.markPreviousJobAsFailed(jobName, jobRepository, previousJobExecution != null ? previousJobExecution.getJobParameters() : jobParameters); - } - else { + } else { jobParameters = CommandLineRunnerUtils.addRunIDToJobParameters(jobName, jobExplorer, jobParameters); } launchJob(jobParameters); } catch (NoJobToExecuteException | NoParametersHaveBeenPassedException | NoPreviousJobExecutionException - | UnknownJobException | JobParametersInvalidException | JobExecutionAlreadyRunningException e) { + | UnknownJobException | JobParametersInvalidException | JobExecutionAlreadyRunningException e) { logger.error(e.getMessage()); logger.debug("Error trace", e); abnormalExit = true; @@ -155,8 +152,7 @@ private void launchJob(JobParameters jobParameters) throws JobExecutionException @Override protected void execute(Job job, JobParameters jobParameters) throws JobExecutionAlreadyRunningException, - JobRestartException, JobInstanceAlreadyCompleteException, JobParametersInvalidException, - JobParametersNotFoundException { + JobRestartException, JobInstanceAlreadyCompleteException, JobParametersInvalidException { logger.info("Running job '" + jobName + "' with parameters: " + jobParameters); super.execute(job, jobParameters); } diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/batch/io/AccessionReportWriterTest.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/batch/io/AccessionReportWriterTest.java index 3aee49e5f..30e5f654c 100644 --- a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/batch/io/AccessionReportWriterTest.java +++ b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/batch/io/AccessionReportWriterTest.java @@ -15,22 +15,20 @@ */ package uk.ac.ebi.eva.accession.pipeline.batch.io; -import org.junit.Before; -import org.junit.Ignore; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; import org.springframework.batch.item.ExecutionContext; import uk.ac.ebi.ampt2d.commons.accession.core.models.AccessionWrapper; - -import uk.ac.ebi.eva.accession.core.service.GetOrCreateAccessionWrapperCreator; -import uk.ac.ebi.eva.accession.core.model.ISubmittedVariant; -import uk.ac.ebi.eva.accession.core.model.SubmittedVariant; +import uk.ac.ebi.eva.accession.core.batch.io.FastaSequenceReader; +import uk.ac.ebi.eva.accession.core.batch.io.FastaSynonymSequenceReader; import uk.ac.ebi.eva.accession.core.contig.ContigMapping; import uk.ac.ebi.eva.accession.core.contig.ContigNaming; import uk.ac.ebi.eva.accession.core.contig.ContigSynonyms; -import uk.ac.ebi.eva.accession.core.batch.io.FastaSequenceReader; -import uk.ac.ebi.eva.accession.core.batch.io.FastaSynonymSequenceReader; +import uk.ac.ebi.eva.accession.core.model.ISubmittedVariant; +import uk.ac.ebi.eva.accession.core.model.SubmittedVariant; +import uk.ac.ebi.eva.accession.core.service.GetOrCreateAccessionWrapperCreator; +import uk.ac.ebi.eva.accession.core.utils.PipelineTemporaryFolderUtil; import uk.ac.ebi.eva.commons.core.models.pipeline.Variant; import uk.ac.ebi.eva.commons.core.models.pipeline.VariantSourceEntry; @@ -45,7 +43,8 @@ import java.util.Arrays; import java.util.Collections; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; import static uk.ac.ebi.eva.accession.pipeline.batch.processors.ContigToGenbankReplacerProcessor.ORIGINAL_CHROMOSOME; public class AccessionReportWriterTest { @@ -112,14 +111,13 @@ public class AccessionReportWriterTest { private ExecutionContext executionContext; - @Rule - public TemporaryFolder temporaryFolderRule = new TemporaryFolder(); - private ContigMapping contigMapping; - @Before + public PipelineTemporaryFolderUtil temporaryFolderUtil = new PipelineTemporaryFolderUtil(); + + @BeforeEach public void setUp() throws Exception { - output = temporaryFolderRule.newFile(); + output = temporaryFolderUtil.newFile(); variantsOutput = new File(output.getAbsolutePath() + AccessionReportWriter.VARIANTS_FILE_SUFFIX); contigsOutput = new File(output.getAbsolutePath() + AccessionReportWriter.CONTIGS_FILE_SUFFIX); Path fastaPath = Paths.get(AccessionReportWriterTest.class.getResource("/input-files/fasta/mock.fa").toURI()); @@ -127,9 +125,9 @@ public void setUp() throws Exception { new ContigSynonyms(CHROMOSOME_1, "assembled-molecule", "1", CONTIG_1, "refseq_1", "chr1", true), new ContigSynonyms("chr2", "assembled-molecule", "2", GENBANK_2, REFSEQ_2, "chr2", false), new ContigSynonyms(SEQUENCE_NAME_3, "unlocalized-scaffold", "1", GENBANK_3, REFSEQ_3, "chr3_random", - true), + true), new ContigSynonyms(SEQUENCE_NAME_4, "unlocalized-scaffold", "4", GENBANK_4, REFSEQ_4, "chr4_random", - true))); + true))); fastaSequenceReader = new FastaSynonymSequenceReader(contigMapping, fastaPath); executionContext = new ExecutionContext(); } @@ -145,25 +143,25 @@ private void writeVariantWithAccessionHelper(int start, String reference, String Variant variant = buildMockVariant(CONTIG_1, CONTIG_1, start, reference, alternate); SubmittedVariant submittedVariant = new SubmittedVariant("accession", TAXONOMY, "project", CONTIG_1, start, - reference, alternate, CLUSTERED_VARIANT, - SUPPORTED_BY_EVIDENCE, MATCHES_ASSEMBLY, ALLELES_MATCH, - VALIDATED, null); + reference, alternate, CLUSTERED_VARIANT, + SUPPORTED_BY_EVIDENCE, MATCHES_ASSEMBLY, ALLELES_MATCH, + VALIDATED, null); AccessionWrapper accessionWrapper = new AccessionWrapper<>(ACCESSION, "1", - submittedVariant); + submittedVariant); AccessionReportWriter accessionReportWriter = new AccessionReportWriter(output, fastaSequenceReader, - contigMapping, - ContigNaming.NO_REPLACEMENT); + contigMapping, + ContigNaming.NO_REPLACEMENT); accessionReportWriter.open(executionContext); accessionReportWriter.write(Collections.singletonList(variant), - GetOrCreateAccessionWrapperCreator.convertToGetOrCreateAccessionWrapper( - Collections.singletonList(accessionWrapper))); + GetOrCreateAccessionWrapperCreator.convertToGetOrCreateAccessionWrapper( + Collections.singletonList(accessionWrapper))); assertEquals(String.join("\t", CONTIG_1, Integer.toString(denormalizedStart), ACCESSION_PREFIX + ACCESSION, - denormalizedReference, denormalizedAlternate, ".", ".", "."), - getFirstVariantLine(variantsOutput)); + denormalizedReference, denormalizedAlternate, ".", ".", "."), + getFirstVariantLine(variantsOutput)); } public static String getFirstVariantLine(File output) throws IOException { @@ -212,33 +210,33 @@ private void writeAndResumeAndWrite(String originalChromosome1, String contig1, SubmittedVariant submittedVariant = new SubmittedVariant("accession", TAXONOMY, "project", contig1, start1, - REFERENCE, ALTERNATE, CLUSTERED_VARIANT, - SUPPORTED_BY_EVIDENCE, MATCHES_ASSEMBLY, ALLELES_MATCH, - VALIDATED, null); + REFERENCE, ALTERNATE, CLUSTERED_VARIANT, + SUPPORTED_BY_EVIDENCE, MATCHES_ASSEMBLY, ALLELES_MATCH, + VALIDATED, null); AccessionWrapper accessionWrapper = new AccessionWrapper<>(ACCESSION, "1", - submittedVariant); + submittedVariant); AccessionReportWriter accessionReportWriter = new AccessionReportWriter(output, fastaSequenceReader, - contigMapping, - ContigNaming.NO_REPLACEMENT); + contigMapping, + ContigNaming.NO_REPLACEMENT); accessionReportWriter.open(executionContext); accessionReportWriter.write(Collections.singletonList(variant), - GetOrCreateAccessionWrapperCreator.convertToGetOrCreateAccessionWrapper( - Collections.singletonList(accessionWrapper))); + GetOrCreateAccessionWrapperCreator.convertToGetOrCreateAccessionWrapper( + Collections.singletonList(accessionWrapper))); accessionReportWriter.close(); // second writer AccessionReportWriter resumingWriter = new AccessionReportWriter(output, fastaSequenceReader, contigMapping, - ContigNaming.NO_REPLACEMENT); + ContigNaming.NO_REPLACEMENT); variant = buildMockVariant(originalChromosome2, contig2, start2, REFERENCE, ALTERNATE); submittedVariant.setContig(contig2); submittedVariant.setStart(start2); resumingWriter.open(executionContext); resumingWriter.write(Collections.singletonList(variant), - GetOrCreateAccessionWrapperCreator.convertToGetOrCreateAccessionWrapper( - Collections.singletonList(accessionWrapper))); + GetOrCreateAccessionWrapperCreator.convertToGetOrCreateAccessionWrapper( + Collections.singletonList(accessionWrapper))); resumingWriter.close(); } @@ -251,7 +249,7 @@ public void resumeWritingWithSameContigs() throws IOException { } @Test - @Ignore("A duplicated variant (two input variants that get the same accession) will appear twice in the VCF " + @Disabled("A duplicated variant (two input variants that get the same accession) will appear twice in the VCF " + "report if they are processed in different batches. There is no easy solution for this, as we would " + "need to cache previous batches. It is even worse if the job fails between the affected batches, as we " + "would need to parse the temporary VCF output or keep some variants in the job repository.") @@ -282,19 +280,19 @@ private void assertContigReplacement(String originalContig, String accessionedCo Variant variant = buildMockVariant(originalContig, accessionedContig, START_1, REFERENCE, ALTERNATE); SubmittedVariant submittedVariant = new SubmittedVariant("accession", TAXONOMY, "project", accessionedContig, - START_1, "", ALTERNATE, CLUSTERED_VARIANT, - SUPPORTED_BY_EVIDENCE, MATCHES_ASSEMBLY, ALLELES_MATCH, - VALIDATED, null); + START_1, "", ALTERNATE, CLUSTERED_VARIANT, + SUPPORTED_BY_EVIDENCE, MATCHES_ASSEMBLY, ALLELES_MATCH, + VALIDATED, null); AccessionWrapper accessionWrapper = new AccessionWrapper<>(ACCESSION, "hash-1", - submittedVariant); + submittedVariant); AccessionReportWriter accessionReportWriter = new AccessionReportWriter(output, fastaSequenceReader, - contigMapping, requestedReplacement); + contigMapping, requestedReplacement); accessionReportWriter.open(new ExecutionContext()); accessionReportWriter.write(Collections.singletonList(variant), - GetOrCreateAccessionWrapperCreator.convertToGetOrCreateAccessionWrapper( - Collections.singletonList(accessionWrapper))); + GetOrCreateAccessionWrapperCreator.convertToGetOrCreateAccessionWrapper( + Collections.singletonList(accessionWrapper))); assertEquals(replacementContig, getFirstVariantLine(variantsOutput).split("\t")[CHROMOSOME_COLUMN_VCF]); } @@ -304,11 +302,11 @@ public void writeChromosomeReplacedFromContig() throws IOException { assertContigReplacement(CHROMOSOME_1, CONTIG_1, ContigNaming.SEQUENCE_NAME, CHROMOSOME_1); } - @Test(expected = IllegalArgumentException.class) + @Test public void writeContigWithoutEquivalent() throws IOException { String contigMissingInAssemblyReport = "contig_missing_in_assembly_report"; - assertContigReplacement(contigMissingInAssemblyReport, contigMissingInAssemblyReport, ContigNaming.SEQUENCE_NAME, - contigMissingInAssemblyReport); + assertThrows(IllegalArgumentException.class, () -> assertContigReplacement(contigMissingInAssemblyReport, contigMissingInAssemblyReport, ContigNaming.SEQUENCE_NAME, + contigMissingInAssemblyReport)); } @Test @@ -324,7 +322,7 @@ public void writeContigWithoutIdenticalReplacement() throws IOException { /** * This test checks that the context base is added from the FASTA, and it doesn't matter which contig naming the * FASTA uses: it's independent of the accessioned contig naming (GenBank) and the VCF report contig naming. - * + *

* Note how in the fasta there's no entry for GENBANK_4 nor SEQUENCE_NAME_4, only for REFSEQ_4 */ @Test diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/batch/io/AccessionWriterTest.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/batch/io/AccessionWriterTest.java index 10fceca99..ab278fe38 100644 --- a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/batch/io/AccessionWriterTest.java +++ b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/batch/io/AccessionWriterTest.java @@ -16,15 +16,13 @@ package uk.ac.ebi.eva.accession.pipeline.batch.io; import org.hamcrest.Matchers; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; -import org.junit.rules.TemporaryFolder; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mockito; import org.springframework.batch.core.JobExecution; +import org.springframework.batch.item.Chunk; import org.springframework.batch.item.ExecutionContext; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; @@ -33,7 +31,7 @@ import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.ampt2d.commons.accession.core.models.AccessionWrapper; import uk.ac.ebi.eva.accession.core.batch.io.FastaSequenceReader; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.SubmittedVariantAccessioningConfiguration; @@ -45,10 +43,13 @@ import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; import uk.ac.ebi.eva.accession.core.service.GetOrCreateAccessionWrapperCreator; import uk.ac.ebi.eva.accession.core.service.nonhuman.SubmittedVariantAccessioningService; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; +import uk.ac.ebi.eva.accession.core.utils.PipelineTemporaryFolderUtil; import uk.ac.ebi.eva.accession.pipeline.batch.processors.VariantConverter; import uk.ac.ebi.eva.accession.pipeline.configuration.InputParametersConfiguration; import uk.ac.ebi.eva.accession.pipeline.configuration.batch.listeners.ListenersConfiguration; import uk.ac.ebi.eva.accession.pipeline.metric.AccessioningMetric; +import uk.ac.ebi.eva.accession.pipeline.test.MongoTestConfiguration; import uk.ac.ebi.eva.commons.core.models.pipeline.Variant; import uk.ac.ebi.eva.commons.core.models.pipeline.VariantSourceEntry; import uk.ac.ebi.eva.metrics.metric.MetricCompute; @@ -68,18 +69,19 @@ import java.util.List; import java.util.stream.Collectors; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertThat; -import static org.junit.Assert.assertTrue; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import static uk.ac.ebi.eva.accession.pipeline.batch.processors.ContigToGenbankReplacerProcessor.ORIGINAL_CHROMOSOME; -@RunWith(SpringRunner.class) +@ExtendWith(SpringExtension.class) @EnableAutoConfiguration @ContextConfiguration(classes = {SubmittedVariantAccessioningConfiguration.class, ListenersConfiguration.class, - InputParametersConfiguration.class}) + InputParametersConfiguration.class, MongoTestConfiguration.class}) @TestPropertySource("classpath:accession-pipeline-test.properties") -public class AccessionWriterTest { +public class AccessionWriterTest extends MongoTestContainerHelper { private static final int TAXONOMY = 3880; @@ -139,11 +141,7 @@ public class AccessionWriterTest { @MockBean private JobExecution jobExecution; - @Rule - public TemporaryFolder temporaryFolderRule = new TemporaryFolder(); - - @Rule - public ExpectedException thrown = ExpectedException.none(); + public PipelineTemporaryFolderUtil temporaryFolderUtil = new PipelineTemporaryFolderUtil(); private AccessionWriter accessionWriter; @@ -153,18 +151,18 @@ public class AccessionWriterTest { private VariantConverter variantConverter; - @Before + @BeforeEach public void setUp() throws Exception { contigMapping = new ContigMapping(Arrays.asList( new ContigSynonyms(CHROMOSOME_1, "assembled-molecule", "1", CONTIG_1, "refseq_1", "chr1", true), new ContigSynonyms(CHROMOSOME_2, "assembled-molecule", "2", CONTIG_2, "refseq_2", "chr2", true))); - output = temporaryFolderRule.newFile(); + output = temporaryFolderUtil.newFile(); variantOutput = new File(output.getAbsolutePath() + AccessionReportWriter.VARIANTS_FILE_SUFFIX); Path fastaPath = Paths.get(AccessionReportWriterTest.class.getResource("/input-files/fasta/mock.fa").toURI()); AccessionReportWriter accessionReportWriter = new AccessionReportWriter(output, - new FastaSequenceReader(fastaPath), - contigMapping, - ContigNaming.SEQUENCE_NAME); + new FastaSequenceReader(fastaPath), + contigMapping, + ContigNaming.SEQUENCE_NAME); variantConverter = new VariantConverter("assembly", TAXONOMY, "project"); accessionWriter = new AccessionWriter(service, accessionReportWriter, variantConverter, metricCompute); accessionReportWriter.open(new ExecutionContext()); @@ -173,8 +171,8 @@ public void setUp() throws Exception { accessionWriter.setJobExecution(jobExecution); } - @After - public void tearDown(){ + @AfterEach + public void tearDown() { metricCompute.clearCount(); } @@ -183,7 +181,7 @@ public void tearDown(){ public void saveSingleAccession() throws Exception { Variant variant = buildMockVariant("contig", START_1); - accessionWriter.write(Collections.singletonList(variant)); + accessionWriter.write(Chunk.of(variant)); ISubmittedVariant submittedVariant = variantConverter.convert(variant); List> accessions = service @@ -204,7 +202,7 @@ public void saveTwoAccession() throws Exception { Variant secondVariant = buildMockVariant("contig", START_2); List variants = Arrays.asList(firstVariant, secondVariant); - accessionWriter.write(variants); + accessionWriter.write(new Chunk<>(variants)); List> accessions = service.get(convert(variants)); assertEquals(2, accessions.size()); @@ -233,7 +231,7 @@ public void variantInsertionCheckOrder() throws Exception { Variant secondVariant = buildMockVariant(CONTIG_1, START_1, "", "A"); List variants = Arrays.asList(firstVariant, secondVariant); - accessionWriter.write(variants); + accessionWriter.write(new Chunk<>(variants)); List> accessions = service.get(convert(variants)); assertEquals(2, accessions.size()); @@ -241,7 +239,7 @@ public void variantInsertionCheckOrder() throws Exception { int firstVariantLineNumber = getVariantLineNumberByPosition(variantOutput, CHROMOSOME_1 + "\t" + START_1); //secondVariant position is START_1 - 1 because it is an insertion and the context base is added int secondVariantLineNumber = getVariantLineNumberByPosition(variantOutput, - CHROMOSOME_1 + "\t" + (START_1 - 1)); + CHROMOSOME_1 + "\t" + (START_1 - 1)); assertTrue(firstVariantLineNumber > secondVariantLineNumber); } @@ -263,7 +261,7 @@ private static int getVariantLineNumberByPosition(File output, String position) public void saveSameAccessionTwice() throws Exception { Variant variant = buildMockVariant("contig", START_1); - accessionWriter.write(Arrays.asList(variant, variant)); + accessionWriter.write(Chunk.of(variant, variant)); ISubmittedVariant submittedVariant = variantConverter.convert(variant); List> accessions = service @@ -281,7 +279,7 @@ public void saveSameAccessionTwice() throws Exception { public void testSaveInitializesCreatedDate() throws Exception { Variant variant = buildMockVariant("contig", START_1); LocalDateTime beforeSave = LocalDateTime.now(); - accessionWriter.write(Collections.singletonList(variant)); + accessionWriter.write(Chunk.of(variant)); LocalDateTime afterSave = LocalDateTime.now(); List> accessions = service @@ -297,7 +295,7 @@ public void testSaveInitializesCreatedDate() throws Exception { public void createAccessionAndItAppearsInTheReportVcf() throws Exception { Variant variant = buildMockVariant("contig", START_1); - accessionWriter.write(Arrays.asList(variant, variant)); + accessionWriter.write(Chunk.of(variant, variant)); List> accessions = service .get(Collections.singletonList(variantConverter.convert(variant))); @@ -305,36 +303,33 @@ public void createAccessionAndItAppearsInTheReportVcf() throws Exception { String vcfLine = AccessionReportWriterTest.getFirstVariantLine(variantOutput); assertEquals(vcfLine.split("\t")[ACCESSION_COLUMN], - ACCESSION_PREFIX + accessions.iterator().next().getAccession()); + ACCESSION_PREFIX + accessions.iterator().next().getAccession()); } @Test public void shouldThrowIfSomeVariantsWereNotAccessioned() { SubmittedVariant variant = new SubmittedVariant("assembly", TAXONOMY, "project", "contig", START_1, - REFERENCE_ALLELE, ALTERNATE_ALLELE, CLUSTERED_VARIANT, false, - MATCHES_ASSEMBLY, ALLELES_MATCH, VALIDATED, null); + REFERENCE_ALLELE, ALTERNATE_ALLELE, CLUSTERED_VARIANT, false, + MATCHES_ASSEMBLY, ALLELES_MATCH, VALIDATED, null); - thrown.expect(IllegalStateException.class); - accessionWriter.checkCountsMatch(Collections.singletonList(variant), new ArrayList<>()); + assertThrows(IllegalStateException.class, () -> accessionWriter.checkCountsMatch(Collections.singletonList(variant), new ArrayList<>())); } @Test public void shouldThrowIfSomeVariantsWereNotAccessionedInAChunkWithRepeatedVariants() { SubmittedVariant firstVariant = new SubmittedVariant("assembly", TAXONOMY, "project", "contig", START_1, - REFERENCE, ALTERNATE, CLUSTERED_VARIANT, false, - MATCHES_ASSEMBLY, ALLELES_MATCH, VALIDATED, null); + REFERENCE, ALTERNATE, CLUSTERED_VARIANT, false, + MATCHES_ASSEMBLY, ALLELES_MATCH, VALIDATED, null); SubmittedVariant secondVariant = new SubmittedVariant("assembly", TAXONOMY, "project", "contig", START_2, - REFERENCE, ALTERNATE, CLUSTERED_VARIANT, false, - MATCHES_ASSEMBLY, ALLELES_MATCH, VALIDATED, null); + REFERENCE, ALTERNATE, CLUSTERED_VARIANT, false, + MATCHES_ASSEMBLY, ALLELES_MATCH, VALIDATED, null); List variants = Arrays.asList(firstVariant, secondVariant, firstVariant, secondVariant); ArrayList> accessions = new ArrayList<>(); accessions.add(new AccessionWrapper<>(EXPECTED_ACCESSION, "hashedMessage", secondVariant)); - thrown.expect(IllegalStateException.class); - accessionWriter.checkCountsMatch(variants, - GetOrCreateAccessionWrapperCreator.convertToGetOrCreateAccessionWrapper( - accessions)); + assertThrows(IllegalStateException.class, () -> accessionWriter.checkCountsMatch(variants, + GetOrCreateAccessionWrapperCreator.convertToGetOrCreateAccessionWrapper(accessions))); } @Test @@ -343,26 +338,26 @@ public void shouldNotThrowIfVariantsDifferOnlyByClusteredVariantAccession() { SubmittedVariant variantWithRs = getSubmittedVariantWithClusteredVariant(NONNULL_CLUSTERED_VARIANT); List variants = Arrays.asList(variant, variant); AccessionWrapper accession = new AccessionWrapper<>(EXPECTED_ACCESSION, - "hashedMessage", - variantWithRs); + "hashedMessage", + variantWithRs); List> accessions = Collections.singletonList(accession); accessionWriter.checkCountsMatch(variants, - GetOrCreateAccessionWrapperCreator.convertToGetOrCreateAccessionWrapper( - accessions)); + GetOrCreateAccessionWrapperCreator.convertToGetOrCreateAccessionWrapper( + accessions)); } @Test public void shouldSortReport() throws Exception { // given List variants = Arrays.asList(buildMockVariant(CONTIG_2, CHROMOSOME_2, START_1), - buildMockVariant(CONTIG_1, CHROMOSOME_1, START_2), - buildMockVariant(CONTIG_3, CHROMOSOME_3, START_1), - buildMockVariant(CONTIG_1, CHROMOSOME_1, START_1), - buildMockVariant(CONTIG_2, CHROMOSOME_2, START_2)); + buildMockVariant(CONTIG_1, CHROMOSOME_1, START_2), + buildMockVariant(CONTIG_3, CHROMOSOME_3, START_1), + buildMockVariant(CONTIG_1, CHROMOSOME_1, START_1), + buildMockVariant(CONTIG_2, CHROMOSOME_2, START_2)); // when - accessionWriter.write(variants); + accessionWriter.write(new Chunk<>(variants)); // then BufferedReader fileInputStream = new BufferedReader(new InputStreamReader(new FileInputStream(variantOutput))); @@ -383,8 +378,8 @@ public void shouldSortReport() throws Exception { private SubmittedVariant getSubmittedVariantWithClusteredVariant(Long clusteredVariant) { return new SubmittedVariant("assembly", TAXONOMY, "project", "contig", START_1, - REFERENCE, ALTERNATE, clusteredVariant, false, - MATCHES_ASSEMBLY, ALLELES_MATCH, VALIDATED, null); + REFERENCE, ALTERNATE, clusteredVariant, false, + MATCHES_ASSEMBLY, ALLELES_MATCH, VALIDATED, null); } private Variant buildMockVariant(String contig, int start) { diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/batch/policies/InvalidVariantSkipPolicyTest.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/batch/policies/InvalidVariantSkipPolicyTest.java index efb92f988..b9dded92f 100644 --- a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/batch/policies/InvalidVariantSkipPolicyTest.java +++ b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/batch/policies/InvalidVariantSkipPolicyTest.java @@ -17,8 +17,8 @@ */ package uk.ac.ebi.eva.accession.pipeline.batch.policies; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import org.mockito.Mock; import org.springframework.batch.item.file.FlatFileParseException; @@ -26,8 +26,8 @@ import uk.ac.ebi.eva.commons.core.models.factories.exception.NonVariantException; import uk.ac.ebi.eva.commons.core.models.pipeline.Variant; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; public class InvalidVariantSkipPolicyTest { @@ -36,7 +36,7 @@ public class InvalidVariantSkipPolicyTest { private InvalidVariantSkipPolicy invalidVariantSkipPolicy; - @Before + @BeforeEach public void setUp() throws Exception { invalidVariantSkipPolicy = new InvalidVariantSkipPolicy(); } diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/batch/processors/ContigToGenbankReplacerProcessorTest.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/batch/processors/ContigToGenbankReplacerProcessorTest.java index 20b185530..5b362d695 100644 --- a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/batch/processors/ContigToGenbankReplacerProcessorTest.java +++ b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/batch/processors/ContigToGenbankReplacerProcessorTest.java @@ -15,8 +15,8 @@ */ package uk.ac.ebi.eva.accession.pipeline.batch.processors; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import uk.ac.ebi.eva.accession.core.contig.ContigMapping; import uk.ac.ebi.eva.commons.core.models.IVariant; import uk.ac.ebi.eva.commons.core.models.pipeline.Variant; @@ -26,15 +26,15 @@ import java.util.Set; import java.util.stream.Collectors; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThrows; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; import static uk.ac.ebi.eva.accession.pipeline.batch.processors.ContigToGenbankReplacerProcessor.ORIGINAL_CHROMOSOME; public class ContigToGenbankReplacerProcessorTest { private ContigToGenbankReplacerProcessor processor; - @Before + @BeforeEach public void setUp() throws Exception { String fileString = ContigToGenbankReplacerProcessorTest.class.getResource( "/input-files/assembly-report/assembly_report.txt").toString(); diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/batch/processors/ExcludeStructuralVariantsProcessorTest.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/batch/processors/ExcludeStructuralVariantsProcessorTest.java index ea221fe01..4b91f4549 100644 --- a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/batch/processors/ExcludeStructuralVariantsProcessorTest.java +++ b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/batch/processors/ExcludeStructuralVariantsProcessorTest.java @@ -17,8 +17,8 @@ */ package uk.ac.ebi.eva.accession.pipeline.batch.processors; -import org.junit.BeforeClass; -import org.junit.Test; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; import uk.ac.ebi.eva.commons.core.models.IVariant; import uk.ac.ebi.eva.commons.core.models.pipeline.Variant; @@ -26,9 +26,9 @@ import java.util.Arrays; import java.util.Objects; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; public class ExcludeStructuralVariantsProcessorTest { @@ -57,7 +57,7 @@ public class ExcludeStructuralVariantsProcessorTest { private static ExcludeStructuralVariantsProcessor processor; - @BeforeClass + @BeforeAll public static void setUp() { processor = new ExcludeStructuralVariantsProcessor(); } diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/batch/processors/VariantConverterTest.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/batch/processors/VariantConverterTest.java index 616c74a7d..b61ccaae3 100644 --- a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/batch/processors/VariantConverterTest.java +++ b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/batch/processors/VariantConverterTest.java @@ -17,15 +17,16 @@ */ package uk.ac.ebi.eva.accession.pipeline.batch.processors; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import uk.ac.ebi.eva.accession.core.model.ISubmittedVariant; import uk.ac.ebi.eva.accession.core.model.SubmittedVariant; import uk.ac.ebi.eva.commons.core.models.pipeline.Variant; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; public class VariantConverterTest { @@ -55,59 +56,59 @@ public class VariantConverterTest { private VariantConverter processor; - @Before + @BeforeEach public void setUp() { processor = new VariantConverter(ASSEMBLY, TAXONOMY, PROJECT); } @Test - public void process() throws Exception { + public void process() { Variant variant = new Variant(CONTIG, START, 1001, REFERENCE_ALLELE, ALTERNATE_ALLELE); ISubmittedVariant processed = processor.convert(variant); ISubmittedVariant expected = new SubmittedVariant(ASSEMBLY, TAXONOMY, PROJECT, CONTIG, START, REFERENCE_ALLELE, - ALTERNATE_ALLELE, CLUSTERED_VARIANT, SUPPORTED_BY_EVIDENCE, - MATCHES_ASSEMBLY, ALLELES_MATCH, VALIDATED, null); + ALTERNATE_ALLELE, CLUSTERED_VARIANT, SUPPORTED_BY_EVIDENCE, + MATCHES_ASSEMBLY, ALLELES_MATCH, VALIDATED, null); assertEquals(expected, processed); assertNull(processed.getCreatedDate()); } - @Test(expected = IllegalArgumentException.class) + @Test public void shouldThrowExceptionAssemblyNull() { - SubmittedVariant submittedVariant = new SubmittedVariant(null, TAXONOMY, PROJECT, CONTIG, START, - REFERENCE_ALLELE, ALTERNATE_ALLELE, CLUSTERED_VARIANT, - SUPPORTED_BY_EVIDENCE, MATCHES_ASSEMBLY, ALLELES_MATCH, - VALIDATED, null); + assertThrows(IllegalArgumentException.class, () -> new SubmittedVariant(null, TAXONOMY, PROJECT, CONTIG, START, + REFERENCE_ALLELE, ALTERNATE_ALLELE, CLUSTERED_VARIANT, + SUPPORTED_BY_EVIDENCE, MATCHES_ASSEMBLY, ALLELES_MATCH, + VALIDATED, null)); } - @Test(expected = IllegalArgumentException.class) + @Test public void shouldThrowExceptionProjectNull() { - SubmittedVariant submittedVariant = new SubmittedVariant(ASSEMBLY, TAXONOMY, null, CONTIG, START, - REFERENCE_ALLELE, ALTERNATE_ALLELE, CLUSTERED_VARIANT, - SUPPORTED_BY_EVIDENCE, MATCHES_ASSEMBLY, ALLELES_MATCH, - VALIDATED, null); + assertThrows(IllegalArgumentException.class, () -> new SubmittedVariant(ASSEMBLY, TAXONOMY, null, CONTIG, START, + REFERENCE_ALLELE, ALTERNATE_ALLELE, CLUSTERED_VARIANT, + SUPPORTED_BY_EVIDENCE, MATCHES_ASSEMBLY, ALLELES_MATCH, + VALIDATED, null)); } - @Test(expected = IllegalArgumentException.class) + @Test public void shouldThrowExceptionContigNull() { - SubmittedVariant submittedVariant = new SubmittedVariant(ASSEMBLY, TAXONOMY, PROJECT, null, START, - REFERENCE_ALLELE, ALTERNATE_ALLELE, CLUSTERED_VARIANT, - SUPPORTED_BY_EVIDENCE, MATCHES_ASSEMBLY, ALLELES_MATCH, - VALIDATED, null); + assertThrows(IllegalArgumentException.class, () -> new SubmittedVariant(ASSEMBLY, TAXONOMY, PROJECT, null, START, + REFERENCE_ALLELE, ALTERNATE_ALLELE, CLUSTERED_VARIANT, + SUPPORTED_BY_EVIDENCE, MATCHES_ASSEMBLY, ALLELES_MATCH, + VALIDATED, null)); } - @Test(expected = IllegalArgumentException.class) + @Test public void shouldThrowExceptionReferenceNull() { - SubmittedVariant submittedVariant = new SubmittedVariant(ASSEMBLY, TAXONOMY, PROJECT, CONTIG, START, null, - ALTERNATE_ALLELE, CLUSTERED_VARIANT, - SUPPORTED_BY_EVIDENCE, MATCHES_ASSEMBLY, ALLELES_MATCH, - VALIDATED, null); + assertThrows(IllegalArgumentException.class, () -> new SubmittedVariant(ASSEMBLY, TAXONOMY, PROJECT, CONTIG, START, null, + ALTERNATE_ALLELE, CLUSTERED_VARIANT, + SUPPORTED_BY_EVIDENCE, MATCHES_ASSEMBLY, ALLELES_MATCH, + VALIDATED, null)); } - @Test(expected = IllegalArgumentException.class) + @Test public void shouldThrowExceptionAlternateNull() { - SubmittedVariant submittedVariant = new SubmittedVariant(ASSEMBLY, TAXONOMY, PROJECT, CONTIG, START, - REFERENCE_ALLELE, null, CLUSTERED_VARIANT, - SUPPORTED_BY_EVIDENCE, MATCHES_ASSEMBLY, ALLELES_MATCH, - VALIDATED, null); + assertThrows(IllegalArgumentException.class, () -> new SubmittedVariant(ASSEMBLY, TAXONOMY, PROJECT, CONTIG, START, + REFERENCE_ALLELE, null, CLUSTERED_VARIANT, + SUPPORTED_BY_EVIDENCE, MATCHES_ASSEMBLY, ALLELES_MATCH, + VALIDATED, null)); } } diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/batch/tasklets/buildReport/BuildReportTaskletTest.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/batch/tasklets/buildReport/BuildReportTaskletTest.java index 279b47038..56189db60 100644 --- a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/batch/tasklets/buildReport/BuildReportTaskletTest.java +++ b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/batch/tasklets/buildReport/BuildReportTaskletTest.java @@ -15,11 +15,9 @@ */ package uk.ac.ebi.eva.accession.pipeline.batch.tasklets.buildReport; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; - +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import uk.ac.ebi.eva.accession.core.utils.PipelineTemporaryFolderUtil; import uk.ac.ebi.eva.accession.pipeline.batch.io.AccessionReportWriter; import java.io.BufferedReader; @@ -32,10 +30,10 @@ import java.nio.file.Files; import java.util.Arrays; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; public class BuildReportTaskletTest { @@ -55,8 +53,7 @@ public class BuildReportTaskletTest { private static final int CHROMOSOME_COLUMN_VCF = 0; - @Rule - public TemporaryFolder temporaryFolderRule = new TemporaryFolder(); + public PipelineTemporaryFolderUtil temporaryFolderUtil = new PipelineTemporaryFolderUtil(); private File output; @@ -64,9 +61,9 @@ public class BuildReportTaskletTest { private File contigsOutput; - @Before + @BeforeEach public void setUp() throws Exception { - output = temporaryFolderRule.newFile(); + output = temporaryFolderUtil.newFile(); variantsOutput = new File(output.getAbsolutePath() + AccessionReportWriter.VARIANTS_FILE_SUFFIX); contigsOutput = new File(output.getAbsolutePath() + AccessionReportWriter.CONTIGS_FILE_SUFFIX); } @@ -97,7 +94,7 @@ private void assertHeaderIsNotWrittenTwice(File output) throws IOException { String variantLine = line; do { - assertFalse("VCF report has header lines after variant lines", variantLine.startsWith("#")); + assertFalse(variantLine.startsWith("#"), "VCF report has header lines after variant lines"); variantLine = fileInputStream.readLine(); } while (variantLine != null); } @@ -131,8 +128,8 @@ private void writeSingleVariantInContigAndVariantFiles(String originalChromosome FileWriter variantsWriter = new FileWriter(variantsOutput); variantsWriter.write(String.join("\t", - Arrays.asList(originalChromosome, START_1.toString(), ACCESSION.toString(), - REFERENCE, ALTERNATE, MISSING, MISSING, MISSING))); + Arrays.asList(originalChromosome, START_1.toString(), ACCESSION.toString(), + REFERENCE, ALTERNATE, MISSING, MISSING, MISSING))); variantsWriter.close(); } diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/batch/tasklets/reportCheck/AccessionWrapperComparatorTest.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/batch/tasklets/reportCheck/AccessionWrapperComparatorTest.java index 923c318ca..68b48ce41 100644 --- a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/batch/tasklets/reportCheck/AccessionWrapperComparatorTest.java +++ b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/batch/tasklets/reportCheck/AccessionWrapperComparatorTest.java @@ -15,12 +15,9 @@ */ package uk.ac.ebi.eva.accession.pipeline.batch.tasklets.reportCheck; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import uk.ac.ebi.ampt2d.commons.accession.core.models.AccessionWrapper; - import uk.ac.ebi.eva.accession.core.model.ISubmittedVariant; import uk.ac.ebi.eva.accession.core.model.SubmittedVariant; import uk.ac.ebi.eva.commons.core.models.pipeline.Variant; @@ -29,7 +26,8 @@ import java.util.List; import java.util.Map; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; public class AccessionWrapperComparatorTest { @@ -51,12 +49,9 @@ public class AccessionWrapperComparatorTest { private AccessionWrapperComparator accessionWrapperComparator; - @Rule - public ExpectedException thrown = ExpectedException.none(); - private Map contigMapping; - @Before + @BeforeEach public void setUp() throws Exception { List variants = Arrays.asList(buildMockVariant(CONTIG_A), buildMockVariant(CONTIG_B), @@ -162,8 +157,8 @@ public void checkUnexpectedContigRaisesException() { buildMockVariant(CONTIG_A)); AccessionWrapperComparator comparator = new AccessionWrapperComparator(variants); - thrown.expect(IllegalStateException.class); - comparator.compare(buildMockAccessionWrapper(CONTIG_11, 100), - buildMockAccessionWrapper(CONTIG_2, 100)); + assertThrows(IllegalStateException.class, () -> + comparator.compare(buildMockAccessionWrapper(CONTIG_11, 100), + buildMockAccessionWrapper(CONTIG_2, 100))); } } diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/batch/tasklets/reportCheck/ReportCheckTaskletTest.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/batch/tasklets/reportCheck/ReportCheckTaskletTest.java index 61dc7f790..a64cf047c 100644 --- a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/batch/tasklets/reportCheck/ReportCheckTaskletTest.java +++ b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/batch/tasklets/reportCheck/ReportCheckTaskletTest.java @@ -15,7 +15,7 @@ */ package uk.ac.ebi.eva.accession.pipeline.batch.tasklets.reportCheck; -import org.junit.Test; +import org.junit.jupiter.api.Test; import org.springframework.batch.core.ExitStatus; import org.springframework.batch.core.JobExecution; import org.springframework.batch.core.StepContribution; @@ -26,10 +26,9 @@ import org.springframework.batch.item.NonTransientResourceException; import org.springframework.batch.item.ParseException; import org.springframework.batch.item.UnexpectedInputException; - +import uk.ac.ebi.eva.accession.core.batch.io.AccessionedVcfLineMapper; import uk.ac.ebi.eva.accession.core.contig.ContigMapping; import uk.ac.ebi.eva.accession.core.contig.ContigSynonyms; -import uk.ac.ebi.eva.accession.core.batch.io.AccessionedVcfLineMapper; import uk.ac.ebi.eva.commons.batch.io.AggregatedVcfReader; import uk.ac.ebi.eva.commons.batch.io.UnwindingItemStreamReader; import uk.ac.ebi.eva.commons.batch.io.VcfReader; @@ -43,7 +42,7 @@ import java.util.Collections; import java.util.Iterator; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.QC_SUBSNP_ACCESSION_STEP; public class ReportCheckTaskletTest { diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSeveralIntervalsOfSubsnpAccessionsJobConfigurationTest.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSeveralIntervalsOfSubsnpAccessionsJobConfigurationTest.java index 6ae36d3c6..b65a1f30f 100644 --- a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSeveralIntervalsOfSubsnpAccessionsJobConfigurationTest.java +++ b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSeveralIntervalsOfSubsnpAccessionsJobConfigurationTest.java @@ -16,10 +16,10 @@ package uk.ac.ebi.eva.accession.pipeline.configuration.batch.jobs; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.batch.core.BatchStatus; import org.springframework.batch.core.JobExecution; import org.springframework.batch.core.StepExecution; @@ -32,16 +32,19 @@ import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import org.springframework.test.web.client.ExpectedCount; import org.springframework.test.web.client.MockRestServiceServer; import org.springframework.web.client.RestTemplate; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.SubmittedVariantAccessioningConfiguration; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; import uk.ac.ebi.eva.accession.core.repository.nonhuman.eva.SubmittedVariantAccessioningRepository; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.pipeline.batch.io.AccessionReportWriter; import uk.ac.ebi.eva.accession.pipeline.parameters.InputParameters; +import uk.ac.ebi.eva.accession.pipeline.test.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.accession.pipeline.test.BatchTestConfiguration; +import uk.ac.ebi.eva.accession.pipeline.test.MongoTestConfiguration; import uk.ac.ebi.eva.commons.core.utils.FileUtils; import uk.ac.ebi.eva.metrics.count.CountServiceParameters; @@ -58,7 +61,7 @@ import java.util.List; import java.util.TreeSet; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.springframework.test.web.client.match.MockRestRequestMatchers.method; import static org.springframework.test.web.client.match.MockRestRequestMatchers.requestTo; import static org.springframework.test.web.client.response.MockRestResponseCreators.withStatus; @@ -67,10 +70,11 @@ import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.SUBSNP_ACCESSION_STEP; import static uk.ac.ebi.eva.accession.pipeline.test.BatchTestConfiguration.JOB_LAUNCHER_SUBSNP_ACCESSION_JOB; -@RunWith(SpringRunner.class) -@ContextConfiguration(classes = {BatchTestConfiguration.class, SubmittedVariantAccessioningConfiguration.class}) +@ExtendWith(SpringExtension.class) +@ContextConfiguration(classes = {BatchTestConfiguration.class, SubmittedVariantAccessioningConfiguration.class, + MongoTestConfiguration.class, BatchJobRepositoryTestConfiguration.class}) @TestPropertySource("classpath:accession-pipeline-interval-test.properties") -public class CreateSeveralIntervalsOfSubsnpAccessionsJobConfigurationTest { +public class CreateSeveralIntervalsOfSubsnpAccessionsJobConfigurationTest extends MongoTestContainerHelper { private static final int EXPECTED_VARIANTS = 22; @@ -98,7 +102,7 @@ public class CreateSeveralIntervalsOfSubsnpAccessionsJobConfigurationTest { private MockRestServiceServer mockServer; - @Before + @BeforeEach public void setUp() throws Exception { this.cleanSlate(); mongoTemplate.dropCollection(SubmittedVariantEntity.class); @@ -108,7 +112,7 @@ public void setUp() throws Exception { .andRespond(withStatus(HttpStatus.OK)); } - @After + @AfterEach public void tearDown() throws Exception { this.cleanSlate(); mongoTemplate.dropCollection(SubmittedVariantEntity.class); diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSubsnpAccessionsJobConfigurationTest.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSubsnpAccessionsJobConfigurationTest.java index 02b005aab..1673ca54a 100644 --- a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSubsnpAccessionsJobConfigurationTest.java +++ b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSubsnpAccessionsJobConfigurationTest.java @@ -16,10 +16,10 @@ package uk.ac.ebi.eva.accession.pipeline.configuration.batch.jobs; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.batch.core.BatchStatus; import org.springframework.batch.core.JobExecution; import org.springframework.batch.core.StepExecution; @@ -32,16 +32,19 @@ import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import org.springframework.test.web.client.ExpectedCount; import org.springframework.test.web.client.MockRestServiceServer; import org.springframework.web.client.RestTemplate; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.SubmittedVariantAccessioningConfiguration; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; import uk.ac.ebi.eva.accession.core.repository.nonhuman.eva.SubmittedVariantAccessioningRepository; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.pipeline.batch.io.AccessionReportWriter; import uk.ac.ebi.eva.accession.pipeline.parameters.InputParameters; +import uk.ac.ebi.eva.accession.pipeline.test.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.accession.pipeline.test.BatchTestConfiguration; +import uk.ac.ebi.eva.accession.pipeline.test.MongoTestConfiguration; import uk.ac.ebi.eva.commons.core.utils.FileUtils; import uk.ac.ebi.eva.metrics.count.CountServiceParameters; @@ -56,8 +59,8 @@ import java.util.Collection; import java.util.Iterator; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.springframework.test.web.client.match.MockRestRequestMatchers.method; import static org.springframework.test.web.client.match.MockRestRequestMatchers.requestTo; import static org.springframework.test.web.client.response.MockRestResponseCreators.withStatus; @@ -66,10 +69,11 @@ import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.SUBSNP_ACCESSION_STEP; import static uk.ac.ebi.eva.accession.pipeline.test.BatchTestConfiguration.JOB_LAUNCHER_SUBSNP_ACCESSION_JOB; -@RunWith(SpringRunner.class) -@ContextConfiguration(classes = {BatchTestConfiguration.class, SubmittedVariantAccessioningConfiguration.class}) +@ExtendWith(SpringExtension.class) +@ContextConfiguration(classes = {BatchTestConfiguration.class, SubmittedVariantAccessioningConfiguration.class, + MongoTestConfiguration.class, BatchJobRepositoryTestConfiguration.class}) @TestPropertySource("classpath:accession-pipeline-test.properties") -public class CreateSubsnpAccessionsJobConfigurationTest { +public class CreateSubsnpAccessionsJobConfigurationTest extends MongoTestContainerHelper { private static final int EXPECTED_VARIANTS = 22; @@ -97,7 +101,7 @@ public class CreateSubsnpAccessionsJobConfigurationTest { private MockRestServiceServer mockServer; - @Before + @BeforeEach public void setUp() throws Exception { this.cleanSlate(); mongoTemplate.dropCollection(SubmittedVariantEntity.class); @@ -107,7 +111,7 @@ public void setUp() throws Exception { .andRespond(withStatus(HttpStatus.OK)); } - @After + @AfterEach public void tearDown() throws Exception { this.cleanSlate(); mongoTemplate.dropCollection(SubmittedVariantEntity.class); diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSubsnpAccessionsRecoverStateTest.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSubsnpAccessionsRecoverStateTest.java index 59f1ab36a..f1e8b3ad1 100644 --- a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSubsnpAccessionsRecoverStateTest.java +++ b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSubsnpAccessionsRecoverStateTest.java @@ -16,26 +16,22 @@ package uk.ac.ebi.eva.accession.pipeline.configuration.batch.jobs; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.batch.core.BatchStatus; import org.springframework.batch.core.JobExecution; import org.springframework.batch.core.StepExecution; import org.springframework.batch.test.JobLauncherTestUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.context.ApplicationContext; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.http.HttpMethod; import org.springframework.http.HttpStatus; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import org.springframework.test.web.client.ExpectedCount; import org.springframework.test.web.client.MockRestServiceServer; import org.springframework.web.client.RestTemplate; @@ -45,10 +41,12 @@ import uk.ac.ebi.eva.accession.core.model.SubmittedVariant; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; import uk.ac.ebi.eva.accession.core.repository.nonhuman.eva.SubmittedVariantAccessioningRepository; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.pipeline.batch.io.AccessionReportWriter; import uk.ac.ebi.eva.accession.pipeline.parameters.InputParameters; +import uk.ac.ebi.eva.accession.pipeline.test.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.accession.pipeline.test.BatchTestConfiguration; -import uk.ac.ebi.eva.accession.pipeline.test.FixSpringMongoDbRule; +import uk.ac.ebi.eva.accession.pipeline.test.MongoTestConfiguration; import uk.ac.ebi.eva.commons.core.utils.FileUtils; import uk.ac.ebi.eva.metrics.count.CountServiceParameters; @@ -62,8 +60,8 @@ import java.util.Iterator; import java.util.List; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.springframework.test.web.client.match.MockRestRequestMatchers.method; import static org.springframework.test.web.client.match.MockRestRequestMatchers.requestTo; import static org.springframework.test.web.client.response.MockRestResponseCreators.withStatus; @@ -72,11 +70,11 @@ import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.SUBSNP_ACCESSION_STEP; import static uk.ac.ebi.eva.accession.pipeline.test.BatchTestConfiguration.JOB_LAUNCHER_SUBSNP_ACCESSION_JOB; -@RunWith(SpringRunner.class) -@ContextConfiguration(classes = {BatchTestConfiguration.class, SubmittedVariantAccessioningConfiguration.class}) +@ExtendWith(SpringExtension.class) +@ContextConfiguration(classes = {BatchTestConfiguration.class, SubmittedVariantAccessioningConfiguration.class, + MongoTestConfiguration.class, BatchJobRepositoryTestConfiguration.class}) @TestPropertySource("classpath:accession-pipeline-recover-state-test.properties") -public class CreateSubsnpAccessionsRecoverStateTest { - private static final String TEST_DB = "test-db"; +public class CreateSubsnpAccessionsRecoverStateTest extends MongoTestContainerHelper { @Autowired private SubmittedVariantAccessioningRepository mongoRepository; @@ -90,14 +88,6 @@ public class CreateSubsnpAccessionsRecoverStateTest { @Autowired private MongoTemplate mongoTemplate; - //needed for @UsingDataSet - @Autowired - private ApplicationContext applicationContext; - - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName(TEST_DB).build()); - @Autowired @Qualifier(JOB_LAUNCHER_SUBSNP_ACCESSION_JOB) private JobLauncherTestUtils jobLauncherTestUtils; @@ -115,8 +105,9 @@ public class CreateSubsnpAccessionsRecoverStateTest { private MockRestServiceServer mockServer; - @Before + @BeforeEach public void setUp() throws Exception { + mongoTemplate.getDb().drop(); this.cleanSlate(); mockServer = MockRestServiceServer.createServer(restTemplate); mockServer.expect(ExpectedCount.manyTimes(), requestTo(new URI(countServiceParameters.getUrl() + URL_PATH_SAVE_COUNT))) @@ -124,10 +115,10 @@ public void setUp() throws Exception { .andRespond(withStatus(HttpStatus.OK)); } - @After + @AfterEach public void tearDown() throws Exception { + mongoTemplate.getDb().drop(); this.cleanSlate(); - mongoTemplate.dropCollection(SubmittedVariantEntity.class); } public void cleanSlate() throws Exception { diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSubsnpAccessionsRecoveringStateJobConfigurationTest.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSubsnpAccessionsRecoveringStateJobConfigurationTest.java index 09fdb091c..78ea91fb3 100644 --- a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSubsnpAccessionsRecoveringStateJobConfigurationTest.java +++ b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSubsnpAccessionsRecoveringStateJobConfigurationTest.java @@ -16,26 +16,22 @@ package uk.ac.ebi.eva.accession.pipeline.configuration.batch.jobs; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.batch.core.BatchStatus; import org.springframework.batch.core.JobExecution; import org.springframework.batch.core.StepExecution; import org.springframework.batch.test.JobLauncherTestUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.context.ApplicationContext; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.http.HttpMethod; import org.springframework.http.HttpStatus; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import org.springframework.test.web.client.ExpectedCount; import org.springframework.test.web.client.MockRestServiceServer; import org.springframework.web.client.RestTemplate; @@ -45,10 +41,12 @@ import uk.ac.ebi.eva.accession.core.model.SubmittedVariant; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; import uk.ac.ebi.eva.accession.core.repository.nonhuman.eva.SubmittedVariantAccessioningRepository; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.pipeline.batch.io.AccessionReportWriter; import uk.ac.ebi.eva.accession.pipeline.parameters.InputParameters; +import uk.ac.ebi.eva.accession.pipeline.test.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.accession.pipeline.test.BatchTestConfiguration; -import uk.ac.ebi.eva.accession.pipeline.test.FixSpringMongoDbRule; +import uk.ac.ebi.eva.accession.pipeline.test.MongoTestConfiguration; import uk.ac.ebi.eva.commons.core.utils.FileUtils; import uk.ac.ebi.eva.metrics.count.CountServiceParameters; @@ -60,7 +58,7 @@ import java.util.Collection; import java.util.Iterator; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.springframework.test.web.client.match.MockRestRequestMatchers.method; import static org.springframework.test.web.client.match.MockRestRequestMatchers.requestTo; import static org.springframework.test.web.client.response.MockRestResponseCreators.withStatus; @@ -69,17 +67,16 @@ import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.SUBSNP_ACCESSION_STEP; import static uk.ac.ebi.eva.accession.pipeline.test.BatchTestConfiguration.JOB_LAUNCHER_SUBSNP_ACCESSION_JOB; -@RunWith(SpringRunner.class) -@ContextConfiguration(classes = {BatchTestConfiguration.class, SubmittedVariantAccessioningConfiguration.class}) +@ExtendWith(SpringExtension.class) +@ContextConfiguration(classes = {BatchTestConfiguration.class, SubmittedVariantAccessioningConfiguration.class, + MongoTestConfiguration.class, BatchJobRepositoryTestConfiguration.class}) @TestPropertySource("classpath:accession-pipeline-recover-test.properties") -public class CreateSubsnpAccessionsRecoveringStateJobConfigurationTest { +public class CreateSubsnpAccessionsRecoveringStateJobConfigurationTest extends MongoTestContainerHelper { public static final long UNCOMMITTED_ACCESSION = 5000000000L; private static final int EXPECTED_VARIANTS_ACCESSIONED_FROM_VCF = 22; - private static final String TEST_DB = "test-db"; - @Autowired private CountServiceParameters countServiceParameters; @@ -103,20 +100,13 @@ public class CreateSubsnpAccessionsRecoveringStateJobConfigurationTest { @Autowired private MongoTemplate mongoTemplate; - //needed for @UsingDataSet - @Autowired - private ApplicationContext applicationContext; - - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName(TEST_DB).build()); - @Autowired @Qualifier(JOB_LAUNCHER_SUBSNP_ACCESSION_JOB) private JobLauncherTestUtils jobLauncherTestUtils; - @Before + @BeforeEach public void setUp() throws Exception { + mongoTemplate.getDb().drop(); this.cleanSlate(); mockServer = MockRestServiceServer.createServer(restTemplate); mockServer.expect(ExpectedCount.manyTimes(), requestTo(new URI(countServiceParameters.getUrl() + URL_PATH_SAVE_COUNT))) @@ -124,10 +114,10 @@ public void setUp() throws Exception { .andRespond(withStatus(HttpStatus.OK)); } - @After + @AfterEach public void tearDown() throws Exception { + mongoTemplate.getDb().drop(); this.cleanSlate(); - mongoTemplate.dropCollection(SubmittedVariantEntity.class); } public void cleanSlate() throws Exception { @@ -143,8 +133,8 @@ public void accessionJobShouldRecoverUncommittedAccessions() throws Exception { runJob(); - assertEquals("The uncommitted accession should be assigned to only 1 object", - 1, repository.findByAccession(UNCOMMITTED_ACCESSION).size()); + assertEquals(1, repository.findByAccession(UNCOMMITTED_ACCESSION).size(), + "The uncommitted accession should be assigned to only 1 object"); assertCountsInMongo(EXPECTED_VARIANTS_ACCESSIONED_FROM_VCF + 1); assertCountsInVcfReport(EXPECTED_VARIANTS_ACCESSIONED_FROM_VCF); diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/DuplicateSSAccQCJobConfigurationTest.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/DuplicateSSAccQCJobConfigurationTest.java index 0348ee6a3..8d40527a4 100644 --- a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/DuplicateSSAccQCJobConfigurationTest.java +++ b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/DuplicateSSAccQCJobConfigurationTest.java @@ -15,30 +15,25 @@ */ package uk.ac.ebi.eva.accession.pipeline.configuration.batch.jobs; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; -import com.mongodb.MongoClient; -import gherkin.deps.com.google.gson.Gson; -import gherkin.deps.com.google.gson.GsonBuilder; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.batch.core.BatchStatus; import org.springframework.batch.core.JobExecution; import org.springframework.batch.test.JobLauncherTestUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.context.ApplicationContext; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpSubmittedVariantEntity; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; +import uk.ac.ebi.eva.accession.pipeline.test.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.accession.pipeline.test.BatchTestConfiguration; -import uk.ac.ebi.eva.accession.pipeline.test.FixSpringMongoDbRule; +import uk.ac.ebi.eva.accession.pipeline.test.MongoTestConfiguration; import java.io.IOException; import java.nio.file.Files; @@ -47,48 +42,34 @@ import java.util.Set; import java.util.stream.Collectors; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.DUPLICATE_SS_ACC_QC_STEP; import static uk.ac.ebi.eva.accession.pipeline.test.BatchTestConfiguration.JOB_LAUNCHER_DUPLICATE_SS_ACC_QC_JOB; -@RunWith(SpringRunner.class) -@ContextConfiguration(classes = {BatchTestConfiguration.class}) +@ExtendWith(SpringExtension.class) +@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class, + BatchJobRepositoryTestConfiguration.class}) @TestPropertySource("classpath:duplicate-ss-acc-qc-test.properties") -public class DuplicateSSAccQCJobConfigurationTest { - - private static final String TEST_DB = "test-db"; +public class DuplicateSSAccQCJobConfigurationTest extends MongoTestContainerHelper { private static final String duplicateSsAccFile = "src/test/resources/duplicateSSAcc.csv"; @Autowired @Qualifier(JOB_LAUNCHER_DUPLICATE_SS_ACC_QC_JOB) private JobLauncherTestUtils jobLauncherTestUtils; - @Autowired - private MongoClient mongoClient; - @Autowired private MongoTemplate mongoTemplate; - //Required by nosql-unit - @Autowired - private ApplicationContext applicationContext; - - private final Gson gson = new GsonBuilder().create(); - - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName(TEST_DB).build()); - - @Before + @BeforeEach public void setUp() throws IOException { - this.mongoClient.dropDatabase(TEST_DB); + mongoTemplate.getDb().drop(); Files.deleteIfExists(Paths.get(duplicateSsAccFile)); } - @After + @AfterEach public void tearDown() throws IOException { - this.mongoClient.dropDatabase(TEST_DB); + mongoTemplate.getDb().drop(); Files.deleteIfExists(Paths.get(duplicateSsAccFile)); } diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/QCSubsnpAccessionsJobConfigurationTest.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/QCSubsnpAccessionsJobConfigurationTest.java index 295c16d0d..9bf30a7ba 100644 --- a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/QCSubsnpAccessionsJobConfigurationTest.java +++ b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/QCSubsnpAccessionsJobConfigurationTest.java @@ -16,8 +16,8 @@ package uk.ac.ebi.eva.accession.pipeline.configuration.batch.jobs; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.batch.core.BatchStatus; import org.springframework.batch.core.ExitStatus; import org.springframework.batch.core.JobExecution; @@ -28,20 +28,24 @@ import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.SubmittedVariantAccessioningConfiguration; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; +import uk.ac.ebi.eva.accession.pipeline.test.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.accession.pipeline.test.BatchTestConfiguration; +import uk.ac.ebi.eva.accession.pipeline.test.MongoTestConfiguration; import java.util.Collection; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.QC_SUBSNP_ACCESSION_STEP; import static uk.ac.ebi.eva.accession.pipeline.test.BatchTestConfiguration.JOB_LAUNCHER_QC_SUBSNP_ACCESSION_JOB; -@RunWith(SpringRunner.class) -@ContextConfiguration(classes = {BatchTestConfiguration.class, SubmittedVariantAccessioningConfiguration.class}) +@ExtendWith(SpringExtension.class) +@ContextConfiguration(classes = {BatchTestConfiguration.class, SubmittedVariantAccessioningConfiguration.class, + MongoTestConfiguration.class, BatchJobRepositoryTestConfiguration.class}) @TestPropertySource("classpath:qc-accession-pipeline-test.properties") -public class QCSubsnpAccessionsJobConfigurationTest { +public class QCSubsnpAccessionsJobConfigurationTest extends MongoTestContainerHelper { @Autowired @Qualifier(JOB_LAUNCHER_QC_SUBSNP_ACCESSION_JOB) private JobLauncherTestUtils jobLauncherTestUtils; diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/CreateSubsnpAccessionsStepConfigurationTest.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/CreateSubsnpAccessionsStepConfigurationTest.java index 19c0c97de..267a39dba 100644 --- a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/CreateSubsnpAccessionsStepConfigurationTest.java +++ b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/CreateSubsnpAccessionsStepConfigurationTest.java @@ -16,10 +16,10 @@ package uk.ac.ebi.eva.accession.pipeline.configuration.batch.steps; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mockito; import org.springframework.batch.core.BatchStatus; import org.springframework.batch.core.ExitStatus; @@ -33,15 +33,18 @@ import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.ampt2d.commons.accession.persistence.jpa.monotonic.service.ContiguousIdBlockService; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.SubmittedVariantAccessioningConfiguration; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; import uk.ac.ebi.eva.accession.core.repository.nonhuman.eva.SubmittedVariantAccessioningRepository; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.pipeline.batch.io.AccessionReportWriter; import uk.ac.ebi.eva.accession.pipeline.batch.io.AccessionWriter; import uk.ac.ebi.eva.accession.pipeline.parameters.InputParameters; +import uk.ac.ebi.eva.accession.pipeline.test.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.accession.pipeline.test.BatchTestConfiguration; +import uk.ac.ebi.eva.accession.pipeline.test.MongoTestConfiguration; import uk.ac.ebi.eva.commons.core.utils.FileUtils; import java.io.FileInputStream; @@ -49,15 +52,16 @@ import java.nio.file.Files; import java.nio.file.Paths; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.Mockito.when; import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.SUBSNP_ACCESSION_STEP; import static uk.ac.ebi.eva.accession.pipeline.test.BatchTestConfiguration.JOB_LAUNCHER_SUBSNP_ACCESSION_JOB; -@RunWith(SpringRunner.class) -@ContextConfiguration(classes = {BatchTestConfiguration.class, SubmittedVariantAccessioningConfiguration.class}) +@ExtendWith(SpringExtension.class) +@ContextConfiguration(classes = {BatchTestConfiguration.class, SubmittedVariantAccessioningConfiguration.class, + MongoTestConfiguration.class, BatchJobRepositoryTestConfiguration.class}) @TestPropertySource("classpath:accession-pipeline-test.properties") -public class CreateSubsnpAccessionsStepConfigurationTest { +public class CreateSubsnpAccessionsStepConfigurationTest extends MongoTestContainerHelper { private static final int EXPECTED_VARIANTS = 22; @@ -85,7 +89,7 @@ public class CreateSubsnpAccessionsStepConfigurationTest { @SpyBean private ContiguousIdBlockService contiguousIdBlockService; - @Before + @BeforeEach public void setUp() throws Exception { // if a new transaction is not created it will fail when(contiguousIdBlockService.getBlockParameters("test-pipeline-ss")) @@ -104,7 +108,7 @@ public void setUp() throws Exception { accessionWriter.setJobExecution(jobExecution); } - @After + @AfterEach public void tearDown() throws Exception { Files.deleteIfExists(Paths.get(inputParameters.getOutputVcf())); Files.deleteIfExists(Paths.get(inputParameters.getOutputVcf() + AccessionReportWriter.VARIANTS_FILE_SUFFIX)); diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/QCSubsnpAccessionsStepConfigurationTest.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/QCSubsnpAccessionsStepConfigurationTest.java index aca599cd9..1737d3d62 100644 --- a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/QCSubsnpAccessionsStepConfigurationTest.java +++ b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/QCSubsnpAccessionsStepConfigurationTest.java @@ -16,9 +16,9 @@ package uk.ac.ebi.eva.accession.pipeline.configuration.batch.steps; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mockito; import org.springframework.batch.core.BatchStatus; import org.springframework.batch.core.ExitStatus; @@ -30,19 +30,23 @@ import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.SubmittedVariantAccessioningConfiguration; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.pipeline.batch.io.AccessionWriter; +import uk.ac.ebi.eva.accession.pipeline.test.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.accession.pipeline.test.BatchTestConfiguration; +import uk.ac.ebi.eva.accession.pipeline.test.MongoTestConfiguration; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.QC_SUBSNP_ACCESSION_STEP; import static uk.ac.ebi.eva.accession.pipeline.test.BatchTestConfiguration.JOB_LAUNCHER_QC_SUBSNP_ACCESSION_JOB; -@RunWith(SpringRunner.class) -@ContextConfiguration(classes = {BatchTestConfiguration.class, SubmittedVariantAccessioningConfiguration.class}) +@ExtendWith(SpringExtension.class) +@ContextConfiguration(classes = {BatchTestConfiguration.class, SubmittedVariantAccessioningConfiguration.class, + MongoTestConfiguration.class, BatchJobRepositoryTestConfiguration.class}) @TestPropertySource("classpath:qc-accession-pipeline-test.properties") -public class QCSubsnpAccessionsStepConfigurationTest { +public class QCSubsnpAccessionsStepConfigurationTest extends MongoTestContainerHelper { @Autowired @Qualifier(JOB_LAUNCHER_QC_SUBSNP_ACCESSION_JOB) private JobLauncherTestUtils jobLauncherTestUtils; @@ -53,7 +57,7 @@ public class QCSubsnpAccessionsStepConfigurationTest { @MockBean private JobExecution jobExecution; - @Before + @BeforeEach public void setUp() throws Exception { Mockito.when(jobExecution.getJobId()).thenReturn(1L); accessionWriter.setJobExecution(jobExecution); diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/runner/EvaAccessionJobLauncherCommandLineRunnerTest.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/runner/EvaAccessionJobLauncherCommandLineRunnerTest.java index f07516469..660cef9cb 100644 --- a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/runner/EvaAccessionJobLauncherCommandLineRunnerTest.java +++ b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/runner/EvaAccessionJobLauncherCommandLineRunnerTest.java @@ -16,13 +16,13 @@ package uk.ac.ebi.eva.accession.pipeline.runner; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Ignore; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.batch.core.JobInstance; import org.springframework.batch.core.StepExecution; import org.springframework.batch.core.explore.JobExplorer; @@ -36,18 +36,20 @@ import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import org.springframework.test.web.client.ExpectedCount; import org.springframework.test.web.client.MockRestServiceServer; import org.springframework.web.client.RestTemplate; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; import uk.ac.ebi.eva.accession.core.runner.CommandLineRunnerUtils; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.pipeline.parameters.InputParameters; +import uk.ac.ebi.eva.accession.pipeline.test.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.accession.pipeline.test.BatchTestConfiguration; +import uk.ac.ebi.eva.accession.pipeline.test.MongoTestConfiguration; import uk.ac.ebi.eva.commons.batch.io.VcfReader; import uk.ac.ebi.eva.metrics.count.CountServiceParameters; -import javax.sql.DataSource; import java.io.File; import java.net.URI; import java.nio.file.Files; @@ -58,7 +60,7 @@ import java.util.Arrays; import java.util.Collections; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.springframework.test.web.client.match.MockRestRequestMatchers.method; import static org.springframework.test.web.client.match.MockRestRequestMatchers.requestTo; import static org.springframework.test.web.client.response.MockRestResponseCreators.withStatus; @@ -72,10 +74,11 @@ import static uk.ac.ebi.eva.accession.pipeline.runner.RunnerUtil.useTempVcfFile; import static uk.ac.ebi.eva.accession.pipeline.runner.RunnerUtil.writeToTempVCFFile; -@RunWith(SpringRunner.class) -@ContextConfiguration(classes = {BatchTestConfiguration.class}) +@ExtendWith(SpringExtension.class) +@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class, + BatchJobRepositoryTestConfiguration.class}) @TestPropertySource("classpath:accession-pipeline-test.properties") -public class EvaAccessionJobLauncherCommandLineRunnerTest { +public class EvaAccessionJobLauncherCommandLineRunnerTest extends MongoTestContainerHelper { @Autowired private InputParameters inputParameters; @@ -86,9 +89,6 @@ public class EvaAccessionJobLauncherCommandLineRunnerTest { @Autowired private JobExplorer jobExplorer; - @Autowired - private DataSource datasource; - @Autowired private EvaAccessionJobLauncherCommandLineRunner runner; @@ -123,18 +123,18 @@ public class EvaAccessionJobLauncherCommandLineRunnerTest { private boolean originalInputParametersCaptured = false; - @BeforeClass + @BeforeAll public static void initializeTempFile() throws Exception { tempVcfInputFileToTestFailingJobs = File.createTempFile("resumeFailingJob", ".vcf.gz"); tempVcfOutputDir = Files.createTempDirectory("contigs_variants_dir"); } - @AfterClass + @AfterAll public static void deleteTempFile() { tempVcfInputFileToTestFailingJobs.delete(); } - @Before + @BeforeEach public void setUp() throws Exception { if (!originalInputParametersCaptured) { originalVcfInputFilePath = inputParameters.getVcf(); @@ -143,8 +143,8 @@ public void setUp() throws Exception { writeToTempVCFFile(originalVcfContent, tempVcfInputFileToTestFailingJobs); originalInputParametersCaptured = true; } - jobRepositoryTestUtils = new JobRepositoryTestUtils(jobRepository, datasource); - runner.setJobNames(SUBSNP_ACCESSION_JOB); + jobRepositoryTestUtils = new JobRepositoryTestUtils(jobRepository); + runner.setJobName(SUBSNP_ACCESSION_JOB); deleteTemporaryContigAndVariantFiles(inputParameters, tempVcfOutputDir); useOriginalVcfFile(inputParameters, originalVcfInputFilePath, vcfReader); @@ -156,7 +156,7 @@ public void setUp() throws Exception { mongoTemplate.dropCollection(SubmittedVariantEntity.class); } - @After + @AfterEach public void tearDown() { jobRepositoryTestUtils.removeJobExecutions(); inputParameters.setForceRestart(false); @@ -171,7 +171,7 @@ public void runJobWithNoErrors() throws Exception { @Test public void runJobWithNoName() throws Exception { - runner.setJobNames(null); + runner.setJobName(null); runner.run(); assertEquals(EvaAccessionJobLauncherCommandLineRunner.EXIT_WITH_ERRORS, runner.getExitCode()); @@ -179,7 +179,7 @@ public void runJobWithNoName() throws Exception { @Test @DirtiesContext - @Ignore + @Disabled public void restartCompletedJobThatIsAlreadyInTheRepository() throws Exception { runner.run(); assertEquals(EvaAccessionJobLauncherCommandLineRunner.EXIT_WITHOUT_ERRORS, runner.getExitCode()); @@ -218,7 +218,7 @@ public void forceRestartButNoJobInTheRepository() throws Exception { @Test @DirtiesContext - @Ignore + @Disabled public void resumeFailingJobFromCorrectChunk() throws Exception { // Jobs A, B, C are run chronological order; A and C have SAME parameters; // A is the job that is run after VCF fault injection (as part of the runTestWithFaultInjection method), diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/runner/JobFailureBlocksReleasedTest.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/runner/JobFailureBlocksReleasedTest.java index d327f8077..cbbb5eebc 100644 --- a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/runner/JobFailureBlocksReleasedTest.java +++ b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/runner/JobFailureBlocksReleasedTest.java @@ -16,29 +16,32 @@ * limitations under the License. */ -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.ampt2d.commons.accession.persistence.jpa.monotonic.repositories.ContiguousIdBlockRepository; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.pipeline.parameters.InputParameters; +import uk.ac.ebi.eva.accession.pipeline.test.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.accession.pipeline.test.BatchTestConfiguration; +import uk.ac.ebi.eva.accession.pipeline.test.MongoTestConfiguration; import uk.ac.ebi.eva.commons.batch.io.VcfReader; import java.io.File; import java.nio.file.Files; import java.nio.file.Path; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.SUBSNP_ACCESSION_JOB; import static uk.ac.ebi.eva.accession.pipeline.runner.RunnerUtil.deleteTemporaryContigAndVariantFiles; import static uk.ac.ebi.eva.accession.pipeline.runner.RunnerUtil.getOriginalVcfContent; @@ -47,10 +50,11 @@ import static uk.ac.ebi.eva.accession.pipeline.runner.RunnerUtil.useTempVcfFile; import static uk.ac.ebi.eva.accession.pipeline.runner.RunnerUtil.writeToTempVCFFile; -@RunWith(SpringRunner.class) -@ContextConfiguration(classes = {BatchTestConfiguration.class}) +@ExtendWith(SpringExtension.class) +@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class, + BatchJobRepositoryTestConfiguration.class}) @TestPropertySource("classpath:accession-pipeline-test.properties") -public class JobFailureBlocksReleasedTest { +public class JobFailureBlocksReleasedTest extends MongoTestContainerHelper { @Autowired private InputParameters inputParameters; @@ -74,24 +78,24 @@ public class JobFailureBlocksReleasedTest { @Autowired private ContiguousIdBlockRepository blockRepository; - @BeforeClass + @BeforeAll public static void initializeTempFile() throws Exception { tempVcfInputFileToTestFailingJobs = File.createTempFile("resumeFailingJob", ".vcf.gz"); tempVcfOutputDir = Files.createTempDirectory("contigs_variants_dir"); } - @AfterClass + @AfterAll public static void deleteTempFile() { tempVcfInputFileToTestFailingJobs.delete(); } - @Before + @BeforeEach public void setUp() throws Exception { originalVcfInputFilePath = inputParameters.getVcf(); originalVcfContent = getOriginalVcfContent(originalVcfInputFilePath); writeToTempVCFFile(originalVcfContent, tempVcfInputFileToTestFailingJobs); - runner.setJobNames(SUBSNP_ACCESSION_JOB); + runner.setJobName(SUBSNP_ACCESSION_JOB); deleteTemporaryContigAndVariantFiles(inputParameters, tempVcfOutputDir); useOriginalVcfFile(inputParameters, originalVcfInputFilePath, vcfReader); diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/runner/RestartFailedJobTest.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/runner/RestartFailedJobTest.java index df57e760e..ea0cd06df 100644 --- a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/runner/RestartFailedJobTest.java +++ b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/runner/RestartFailedJobTest.java @@ -16,12 +16,12 @@ package uk.ac.ebi.eva.accession.pipeline.runner; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.batch.core.JobInstance; import org.springframework.batch.core.StepExecution; import org.springframework.batch.core.explore.JobExplorer; @@ -36,26 +36,28 @@ import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import org.springframework.test.web.client.ExpectedCount; import org.springframework.test.web.client.MockRestServiceServer; import org.springframework.web.client.RestTemplate; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; import uk.ac.ebi.eva.accession.core.runner.CommandLineRunnerUtils; import uk.ac.ebi.eva.accession.core.service.nonhuman.SubmittedVariantAccessioningService; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.pipeline.parameters.InputParameters; +import uk.ac.ebi.eva.accession.pipeline.test.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.accession.pipeline.test.BatchTestConfiguration; +import uk.ac.ebi.eva.accession.pipeline.test.MongoTestConfiguration; import uk.ac.ebi.eva.commons.batch.io.VcfReader; import uk.ac.ebi.eva.metrics.count.CountServiceParameters; -import javax.sql.DataSource; import java.io.File; import java.net.URI; import java.nio.file.Files; import java.nio.file.Path; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.mockito.Mockito.doNothing; import static org.springframework.test.web.client.match.MockRestRequestMatchers.method; import static org.springframework.test.web.client.match.MockRestRequestMatchers.requestTo; @@ -70,10 +72,11 @@ import static uk.ac.ebi.eva.accession.pipeline.runner.RunnerUtil.useTempVcfFile; import static uk.ac.ebi.eva.accession.pipeline.runner.RunnerUtil.writeToTempVCFFile; -@RunWith(SpringRunner.class) -@ContextConfiguration(classes = {BatchTestConfiguration.class}) +@ExtendWith(SpringExtension.class) +@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class, + BatchJobRepositoryTestConfiguration.class}) @TestPropertySource("classpath:accession-pipeline-test.properties") -public class RestartFailedJobTest { +public class RestartFailedJobTest extends MongoTestContainerHelper { @Autowired private InputParameters inputParameters; @@ -84,9 +87,6 @@ public class RestartFailedJobTest { @Autowired private JobExplorer jobExplorer; - @Autowired - private DataSource datasource; - @Autowired private EvaAccessionJobLauncherCommandLineRunner runner; @@ -115,8 +115,6 @@ public class RestartFailedJobTest { private static String originalVcfInputFilePath; - private static String originalVcfOutputFilePath; - private static String originalVcfContent; private boolean originalInputParametersCaptured = false; @@ -124,28 +122,27 @@ public class RestartFailedJobTest { @SpyBean private SubmittedVariantAccessioningService accessioningServiceSpy; - @BeforeClass + @BeforeAll public static void initializeTempFile() throws Exception { tempVcfInputFileToTestFailingJobs = File.createTempFile("resumeFailingJob", ".vcf.gz"); tempVcfOutputDir = Files.createTempDirectory("contigs_variants_dir"); } - @AfterClass + @AfterAll public static void deleteTempFile() { tempVcfInputFileToTestFailingJobs.delete(); } - @Before + @BeforeEach public void setUp() throws Exception { if (!originalInputParametersCaptured) { originalVcfInputFilePath = inputParameters.getVcf(); - originalVcfOutputFilePath = inputParameters.getOutputVcf(); originalVcfContent = getOriginalVcfContent(originalVcfInputFilePath); writeToTempVCFFile(originalVcfContent, tempVcfInputFileToTestFailingJobs); originalInputParametersCaptured = true; } - jobRepositoryTestUtils = new JobRepositoryTestUtils(jobRepository, datasource); - runner.setJobNames(SUBSNP_ACCESSION_JOB); + jobRepositoryTestUtils = new JobRepositoryTestUtils(jobRepository); + runner.setJobName(SUBSNP_ACCESSION_JOB); deleteTemporaryContigAndVariantFiles(inputParameters, tempVcfOutputDir); useOriginalVcfFile(inputParameters, originalVcfInputFilePath, vcfReader); @@ -160,7 +157,7 @@ public void setUp() throws Exception { } - @After + @AfterEach public void tearDown() { jobRepositoryTestUtils.removeJobExecutions(); inputParameters.setForceRestart(false); diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/runner/SSAccessionRecoveryTest.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/runner/SSAccessionRecoveryTest.java index 16dbca827..dacbf692c 100644 --- a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/runner/SSAccessionRecoveryTest.java +++ b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/runner/SSAccessionRecoveryTest.java @@ -16,33 +16,37 @@ * limitations under the License. */ -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.batch.test.context.SpringBatchTest; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.ampt2d.commons.accession.persistence.jpa.monotonic.entities.ContiguousIdBlock; import uk.ac.ebi.ampt2d.commons.accession.persistence.jpa.monotonic.repositories.ContiguousIdBlockRepository; import uk.ac.ebi.eva.accession.core.model.SubmittedVariant; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; import uk.ac.ebi.eva.accession.core.repository.nonhuman.eva.SubmittedVariantAccessioningRepository; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; +import uk.ac.ebi.eva.accession.pipeline.test.BatchJobRepositoryTestConfiguration; +import uk.ac.ebi.eva.accession.pipeline.test.MongoTestConfiguration; import uk.ac.ebi.eva.accession.pipeline.test.SSAccessionRecoveryTestConfiguration; import java.util.ArrayList; import java.util.List; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.SS_ACCESSION_RECOVERY_JOB; -@RunWith(SpringRunner.class) -@ContextConfiguration(classes = {SSAccessionRecoveryTestConfiguration.class}) +@ExtendWith(SpringExtension.class) +@ContextConfiguration(classes = {SSAccessionRecoveryTestConfiguration.class, MongoTestConfiguration.class, + BatchJobRepositoryTestConfiguration.class}) @TestPropertySource("classpath:ss-accession-recovery.properties") @SpringBatchTest -public class SSAccessionRecoveryTest { +public class SSAccessionRecoveryTest extends MongoTestContainerHelper { @Autowired private EvaAccessionJobLauncherCommandLineRunner runner; @@ -59,7 +63,7 @@ public void testContiguousBlocksForCategorySSAreRecovered() throws Exception { verifyInitialDBState(); // recovery cut off time is -14 days (provided in ss-accession-recovery.properties) - runner.setJobNames(SS_ACCESSION_RECOVERY_JOB); + runner.setJobName(SS_ACCESSION_RECOVERY_JOB); runner.run(); assertEquals(EvaAccessionJobLauncherCommandLineRunner.EXIT_WITHOUT_ERRORS, runner.getExitCode()); diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/test/BatchJobRepositoryTestConfiguration.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/test/BatchJobRepositoryTestConfiguration.java new file mode 100644 index 000000000..776d64a87 --- /dev/null +++ b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/test/BatchJobRepositoryTestConfiguration.java @@ -0,0 +1,22 @@ +package uk.ac.ebi.eva.accession.pipeline.test; + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Primary; +import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseBuilder; +import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseType; + +import javax.sql.DataSource; + +@Configuration +public class BatchJobRepositoryTestConfiguration { + + @Bean + @Primary + public DataSource dataSource() { + return new EmbeddedDatabaseBuilder() + .setType(EmbeddedDatabaseType.HSQL) + .addScript("org/springframework/batch/core/schema-hsqldb.sql") + .build(); + } +} \ No newline at end of file diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/test/BatchTestConfiguration.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/test/BatchTestConfiguration.java index 132d38828..38a6459e5 100644 --- a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/test/BatchTestConfiguration.java +++ b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/test/BatchTestConfiguration.java @@ -17,15 +17,16 @@ package uk.ac.ebi.eva.accession.pipeline.test; import org.springframework.batch.core.Job; +import org.springframework.batch.core.launch.JobLauncher; +import org.springframework.batch.core.repository.JobRepository; import org.springframework.batch.test.JobLauncherTestUtils; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Import; import uk.ac.ebi.eva.accession.pipeline.configuration.batch.io.AccessionWriterConfiguration; -import uk.ac.ebi.eva.accession.pipeline.configuration.batch.io.SSAccFileReaderConfiguration; import uk.ac.ebi.eva.accession.pipeline.configuration.batch.io.DuplicateSSAccQCWriterConfiguration; +import uk.ac.ebi.eva.accession.pipeline.configuration.batch.io.SSAccFileReaderConfiguration; import uk.ac.ebi.eva.accession.pipeline.configuration.batch.io.VcfReaderConfiguration; import uk.ac.ebi.eva.accession.pipeline.configuration.batch.jobs.DuplicateSSAccQCJobConfiguration; import uk.ac.ebi.eva.accession.pipeline.configuration.batch.jobs.QCSubsnpAccessionsJobConfiguration; @@ -63,36 +64,33 @@ public class BatchTestConfiguration { public static final String JOB_LAUNCHER_DUPLICATE_SS_ACC_QC_JOB = "JOB_LAUNCHER_DUPLICATE_SS_ACC_QC_JOB"; @Bean(JOB_LAUNCHER_SUBSNP_ACCESSION_JOB) - public JobLauncherTestUtils jobLauncherTestUtilsCreate() { - return new JobLauncherTestUtils() { - @Override - @Autowired - public void setJob(@Qualifier(SUBSNP_ACCESSION_JOB) Job job) { - super.setJob(job); - } - }; + public JobLauncherTestUtils jobLauncherTestUtilsCreate(JobLauncher jobLauncher, JobRepository jobRepository, + @Qualifier(SUBSNP_ACCESSION_JOB) Job job) { + JobLauncherTestUtils utils = new JobLauncherTestUtils(); + utils.setJobLauncher(jobLauncher); + utils.setJobRepository(jobRepository); + utils.setJob(job); + return utils; } @Bean(JOB_LAUNCHER_QC_SUBSNP_ACCESSION_JOB) - public JobLauncherTestUtils jobLauncherTestUtilsQC() { - return new JobLauncherTestUtils() { - @Override - @Autowired - public void setJob(@Qualifier(QC_SUBSNP_ACCESSION_JOB) Job job) { - super.setJob(job); - } - }; + public JobLauncherTestUtils jobLauncherTestUtilsQC(JobLauncher jobLauncher, JobRepository jobRepository, + @Qualifier(QC_SUBSNP_ACCESSION_JOB) Job job) { + JobLauncherTestUtils utils = new JobLauncherTestUtils(); + utils.setJobLauncher(jobLauncher); + utils.setJobRepository(jobRepository); + utils.setJob(job); + return utils; } @Bean(JOB_LAUNCHER_DUPLICATE_SS_ACC_QC_JOB) - public JobLauncherTestUtils jobLauncherTestUtilsDuplicateSSAccQC() { - return new JobLauncherTestUtils() { - @Override - @Autowired - public void setJob(@Qualifier(DUPLICATE_SS_ACC_QC_JOB) Job job) { - super.setJob(job); - } - }; + public JobLauncherTestUtils jobLauncherTestUtilsDuplicateSSAccQC(JobLauncher jobLauncher, JobRepository jobRepository, + @Qualifier(DUPLICATE_SS_ACC_QC_JOB) Job job) { + JobLauncherTestUtils utils = new JobLauncherTestUtils(); + utils.setJobLauncher(jobLauncher); + utils.setJobRepository(jobRepository); + utils.setJob(job); + return utils; } @Bean diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/test/FixSpringMongoDbRule.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/test/FixSpringMongoDbRule.java deleted file mode 100644 index 0ff4b7dca..000000000 --- a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/test/FixSpringMongoDbRule.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * - * Copyright 2019 EMBL - European Bioinformatics Institute - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -package uk.ac.ebi.eva.accession.pipeline.test; - -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfiguration; -import com.lordofthejars.nosqlunit.mongodb.SpringMongoDbRule; - -/** - * Temporary fix until nosql unit rc-6 or final is released - */ -public class FixSpringMongoDbRule extends SpringMongoDbRule { - - public FixSpringMongoDbRule(MongoDbConfiguration mongoDbConfiguration) { - super(mongoDbConfiguration); - } - - public FixSpringMongoDbRule(MongoDbConfiguration mongoDbConfiguration, Object object) { - super(mongoDbConfiguration, object); - } - - @Override - public void close() { - // DO NOT CLOSE the connection (Spring will do it when destroying the context) - } - -} diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/test/MongoTestConfiguration.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/test/MongoTestConfiguration.java new file mode 100644 index 000000000..2a261f1b6 --- /dev/null +++ b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/test/MongoTestConfiguration.java @@ -0,0 +1,102 @@ +/* + * Copyright 2021 EMBL - European Bioinformatics Institute + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package uk.ac.ebi.eva.accession.pipeline.test; + +import com.mongodb.ReadPreference; +import com.mongodb.WriteConcern; +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoClients; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.domain.EntityScan; +import org.springframework.boot.test.autoconfigure.data.mongo.AutoConfigureDataMongo; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Primary; +import org.springframework.data.mongodb.MongoDatabaseFactory; +import org.springframework.data.mongodb.config.EnableMongoAuditing; +import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.data.mongodb.core.SimpleMongoClientDatabaseFactory; +import org.springframework.data.mongodb.core.WriteResultChecking; +import org.springframework.data.mongodb.core.convert.DefaultDbRefResolver; +import org.springframework.data.mongodb.core.convert.DefaultMongoTypeMapper; +import org.springframework.data.mongodb.core.convert.MappingMongoConverter; +import org.springframework.data.mongodb.core.convert.MongoCustomConversions; +import org.springframework.data.mongodb.core.mapping.MongoMappingContext; +import org.springframework.data.mongodb.repository.config.EnableMongoRepositories; + +import java.util.Collections; + +@Configuration +@EntityScan(basePackages = {"uk.ac.ebi.eva.accession.core.persistence"}) +@EnableMongoRepositories(basePackages = { + "uk.ac.ebi.eva.accession.core.persistence", + "uk.ac.ebi.ampt2d.commons.accession.persistence.mongodb.repository"}) +@EnableMongoAuditing +@AutoConfigureDataMongo +public class MongoTestConfiguration { + + @Value("${spring.data.mongodb.uri}") + private String mongoUri; + + @Value("${spring.data.mongodb.database}") + private String database; + + @Value("${mongodb.read-preference:primary}") + private String readPreference; + + @Bean + @Primary + public MongoClient mongoClient() { + return MongoClients.create(mongoUri); + } + + @Bean("primaryFactory") + @Primary + public MongoDatabaseFactory mongoDbFactory() { + return new SimpleMongoClientDatabaseFactory(mongoClient(), database); + } + + @Bean + @Primary + public MongoCustomConversions mongoCustomConversions() { + return new MongoCustomConversions(Collections.emptyList()); + } + + @Bean + @Primary + public MongoMappingContext mongoMappingContext(MongoCustomConversions conversions) { + MongoMappingContext context = new MongoMappingContext(); + context.setSimpleTypeHolder(conversions.getSimpleTypeHolder()); + context.setAutoIndexCreation(false); + return context; + } + + @Bean + @Primary + public MongoTemplate mongoTemplate(MongoMappingContext mongoMappingContext, + MongoCustomConversions conversions) { + MappingMongoConverter converter = new MappingMongoConverter( + new DefaultDbRefResolver(mongoDbFactory()), mongoMappingContext); + converter.setCustomConversions(conversions); + converter.setTypeMapper(new DefaultMongoTypeMapper(null)); + converter.afterPropertiesSet(); + MongoTemplate mongoTemplate = new MongoTemplate(mongoDbFactory(), converter); + mongoTemplate.setReadPreference(ReadPreference.valueOf(readPreference)); + mongoTemplate.setWriteConcern(WriteConcern.MAJORITY); + mongoTemplate.setWriteResultChecking(WriteResultChecking.EXCEPTION); + return mongoTemplate; + } +} diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/test/SSAccessionRecoveryTestConfiguration.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/test/SSAccessionRecoveryTestConfiguration.java index 4e9d3a2b5..4556a4439 100644 --- a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/test/SSAccessionRecoveryTestConfiguration.java +++ b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/test/SSAccessionRecoveryTestConfiguration.java @@ -16,7 +16,8 @@ package uk.ac.ebi.eva.accession.pipeline.test; -import org.springframework.batch.core.configuration.annotation.BatchConfigurer; +import org.springframework.batch.core.launch.JobLauncher; +import org.springframework.batch.core.repository.JobRepository; import org.springframework.batch.test.JobLauncherTestUtils; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.context.annotation.Bean; @@ -28,12 +29,8 @@ import uk.ac.ebi.eva.accession.pipeline.configuration.batch.recovery.SSAccessionRecoveryServiceConfiguration; import uk.ac.ebi.eva.accession.pipeline.configuration.batch.steps.SSAccessionRecoveryStepConfiguration; import uk.ac.ebi.eva.accession.pipeline.runner.EvaAccessionJobLauncherCommandLineRunner; -import uk.ac.ebi.eva.commons.batch.configuration.SpringBoot1CompatibilityConfiguration; import uk.ac.ebi.eva.commons.batch.job.JobExecutionApplicationListener; -import javax.persistence.EntityManagerFactory; -import javax.sql.DataSource; - @EnableAutoConfiguration @Import({SSAccessionRecoveryJobConfiguration.class, SSAccessionRecoveryStepConfiguration.class, @@ -43,17 +40,10 @@ EvaAccessionJobLauncherCommandLineRunner.class}) public class SSAccessionRecoveryTestConfiguration { @Bean - public BatchConfigurer configurer(DataSource dataSource, EntityManagerFactory entityManagerFactory) - throws Exception { - return SpringBoot1CompatibilityConfiguration.getSpringBoot1CompatibleBatchConfigurer(dataSource, - entityManagerFactory); - } - - @Bean - public JobLauncherTestUtils jobLauncherTestUtils(BatchConfigurer configurer) throws Exception { + public JobLauncherTestUtils jobLauncherTestUtils(JobLauncher jobLauncher, JobRepository jobRepository) { JobLauncherTestUtils jobLauncherTestUtils = new JobLauncherTestUtils(); - jobLauncherTestUtils.setJobLauncher(configurer.getJobLauncher()); - jobLauncherTestUtils.setJobRepository(configurer.getJobRepository()); + jobLauncherTestUtils.setJobLauncher(jobLauncher); + jobLauncherTestUtils.setJobRepository(jobRepository); return jobLauncherTestUtils; } diff --git a/eva-accession-pipeline/src/test/resources/properties/accession-pipeline-interval-test.properties b/eva-accession-pipeline/src/test/resources/properties/accession-pipeline-interval-test.properties index 2c6b400e8..b30fb3297 100644 --- a/eva-accession-pipeline/src/test/resources/properties/accession-pipeline-interval-test.properties +++ b/eva-accession-pipeline/src/test/resources/properties/accession-pipeline-interval-test.properties @@ -23,9 +23,6 @@ parameters.contigNaming=SEQUENCE_NAME spring.jpa.show-sql=true spring.data.mongodb.database=test-db -spring.data.mongodb.host=|eva.mongo.host.test| -spring.data.mongodb.password= -spring.data.mongodb.port=27017 mongodb.read-preference=primary # See https://github.com/spring-projects/spring-boot/wiki/Spring-Boot-2.1-Release-Notes#bean-overriding diff --git a/eva-accession-pipeline/src/test/resources/properties/accession-pipeline-recover-state-test.properties b/eva-accession-pipeline/src/test/resources/properties/accession-pipeline-recover-state-test.properties index 8062413e2..2edb00288 100644 --- a/eva-accession-pipeline/src/test/resources/properties/accession-pipeline-recover-state-test.properties +++ b/eva-accession-pipeline/src/test/resources/properties/accession-pipeline-recover-state-test.properties @@ -8,9 +8,10 @@ spring.datasource.driver-class-name=org.hsqldb.jdbcDriver spring.datasource.url=jdbc:hsqldb:mem:db;sql.syntax_pgs=true;DB_CLOSE_DELAY=-1 spring.datasource.username=SA spring.datasource.password= -spring.datasource.schema=test-data/contiguous_id_blocks_schema.sql -spring.datasource.data=test-data/contiguous_id_blocks_recover_state_data.sql -spring.jpa.hibernate.ddl-auto=update +spring.sql.init.schema-locations=classpath:test-data/contiguous_id_blocks_schema.sql +spring.sql.init.data-locations=classpath:test-data/contiguous_id_blocks_recover_state_data.sql +spring.sql.init.mode=always +spring.jpa.hibernate.ddl-auto=none parameters.assemblyAccession=assembly parameters.taxonomyAccession=1111 @@ -30,11 +31,7 @@ eva.count-stats.password=password spring.jpa.show-sql=true -spring.data.mongodb.uri=mongodb://|eva.mongo.host.test|:27017 spring.data.mongodb.database=test-db -spring.data.mongodb.host=|eva.mongo.host.test| -spring.data.mongodb.password= -spring.data.mongodb.port=27017 mongodb.read-preference=primary # See https://github.com/spring-projects/spring-boot/wiki/Spring-Boot-2.1-Release-Notes#bean-overriding diff --git a/eva-accession-pipeline/src/test/resources/properties/accession-pipeline-recover-test.properties b/eva-accession-pipeline/src/test/resources/properties/accession-pipeline-recover-test.properties index ca993b4bb..26d8215a4 100644 --- a/eva-accession-pipeline/src/test/resources/properties/accession-pipeline-recover-test.properties +++ b/eva-accession-pipeline/src/test/resources/properties/accession-pipeline-recover-test.properties @@ -12,9 +12,10 @@ spring.datasource.driver-class-name=org.hsqldb.jdbcDriver spring.datasource.url=jdbc:hsqldb:mem:db;sql.syntax_pgs=true;DB_CLOSE_DELAY=-1 spring.datasource.username=SA spring.datasource.password= -spring.datasource.schema=test-data/contiguous_id_blocks_schema.sql -spring.datasource.data=test-data/contiguous_id_blocks_data.sql -spring.jpa.hibernate.ddl-auto=update +spring.sql.init.schema-locations=classpath:test-data/contiguous_id_blocks_schema.sql +spring.sql.init.data-locations=classpath:test-data/contiguous_id_blocks_data.sql +spring.sql.init.mode=always +spring.jpa.hibernate.ddl-auto=none parameters.assemblyAccession=assembly parameters.taxonomyAccession=1111 @@ -30,11 +31,7 @@ parameters.contigNaming=SEQUENCE_NAME spring.jpa.show-sql=true -spring.data.mongodb.uri=mongodb://|eva.mongo.host.test|:27017 spring.data.mongodb.database=test-db -spring.data.mongodb.host=|eva.mongo.host.test| -spring.data.mongodb.password= -spring.data.mongodb.port=27017 mongodb.read-preference=primary # See https://github.com/spring-projects/spring-boot/wiki/Spring-Boot-2.1-Release-Notes#bean-overriding diff --git a/eva-accession-pipeline/src/test/resources/properties/accession-pipeline-test.properties b/eva-accession-pipeline/src/test/resources/properties/accession-pipeline-test.properties index 7ad05b90b..5a20cf25b 100644 --- a/eva-accession-pipeline/src/test/resources/properties/accession-pipeline-test.properties +++ b/eva-accession-pipeline/src/test/resources/properties/accession-pipeline-test.properties @@ -15,6 +15,7 @@ parameters.projectAccession=project parameters.chunkSize=5 parameters.vcf=src/test/resources/input-files/vcf/small_genotyped.vcf.gz parameters.vcfAggregation=NONE +parameters.aggregatedMappingFile=src/test/resources/output_files/aggregated_mapping_file.txt parameters.fasta=src/test/resources/input-files/fasta/Homo_sapiens.GRCh37.75.chr20.head_1200.fa parameters.outputVcf=/tmp/accession-output.vcf @@ -24,9 +25,6 @@ parameters.contigNaming=SEQUENCE_NAME spring.jpa.show-sql=true spring.data.mongodb.database=test-db -spring.data.mongodb.host=|eva.mongo.host.test| -spring.data.mongodb.password= -spring.data.mongodb.port=27017 mongodb.read-preference=primary # See https://github.com/spring-projects/spring-boot/wiki/Spring-Boot-2.1-Release-Notes#bean-overriding diff --git a/eva-accession-pipeline/src/test/resources/properties/duplicate-ss-acc-qc-test.properties b/eva-accession-pipeline/src/test/resources/properties/duplicate-ss-acc-qc-test.properties index 3a37481af..183cdbf90 100644 --- a/eva-accession-pipeline/src/test/resources/properties/duplicate-ss-acc-qc-test.properties +++ b/eva-accession-pipeline/src/test/resources/properties/duplicate-ss-acc-qc-test.properties @@ -25,9 +25,6 @@ parameters.contigNaming=SEQUENCE_NAME spring.jpa.show-sql=true spring.data.mongodb.database=test-db -spring.data.mongodb.host=|eva.mongo.host.test| -spring.data.mongodb.password= -spring.data.mongodb.port=27017 mongodb.read-preference=primary # See https://github.com/spring-projects/spring-boot/wiki/Spring-Boot-2.1-Release-Notes#bean-overriding diff --git a/eva-accession-pipeline/src/test/resources/properties/qc-accession-pipeline-test.properties b/eva-accession-pipeline/src/test/resources/properties/qc-accession-pipeline-test.properties index da55ada11..28c3df942 100644 --- a/eva-accession-pipeline/src/test/resources/properties/qc-accession-pipeline-test.properties +++ b/eva-accession-pipeline/src/test/resources/properties/qc-accession-pipeline-test.properties @@ -1,5 +1,9 @@ accessioning.submitted.categoryId=test-pipeline-ss +eva.count-stats.url=http://localhost:8080 +eva.count-stats.username=username +eva.count-stats.password=password + accessioning.monotonic.test-pipeline-ss.blockSize=100000 accessioning.monotonic.test-pipeline-ss.blockStartValue=5000000000 accessioning.monotonic.test-pipeline-ss.nextBlockInterval=1000000000 @@ -19,9 +23,6 @@ parameters.contigNaming=SEQUENCE_NAME spring.jpa.show-sql=true spring.data.mongodb.database=test-db -spring.data.mongodb.host=|eva.mongo.host.test| -spring.data.mongodb.password= -spring.data.mongodb.port=27017 mongodb.read-preference=primary # See https://github.com/spring-projects/spring-boot/wiki/Spring-Boot-2.1-Release-Notes#bean-overriding diff --git a/eva-accession-pipeline/src/test/resources/properties/ss-accession-recovery.properties b/eva-accession-pipeline/src/test/resources/properties/ss-accession-recovery.properties index a16c85174..343cf5fde 100644 --- a/eva-accession-pipeline/src/test/resources/properties/ss-accession-recovery.properties +++ b/eva-accession-pipeline/src/test/resources/properties/ss-accession-recovery.properties @@ -4,9 +4,10 @@ spring.datasource.driver-class-name=org.hsqldb.jdbcDriver spring.datasource.url=jdbc:hsqldb:mem:db;sql.syntax_pgs=true;DB_CLOSE_DELAY=-1 spring.datasource.username=SA spring.datasource.password= -spring.datasource.schema=test-data/contiguous_id_blocks_schema.sql -spring.datasource.data=test-data/ss_accession_recovery_test_data.sql -spring.jpa.hibernate.ddl-auto=update +spring.sql.init.schema-locations=classpath:test-data/contiguous_id_blocks_schema.sql +spring.sql.init.data-locations=classpath:test-data/ss_accession_recovery_test_data.sql +spring.sql.init.mode=always +spring.jpa.hibernate.ddl-auto=none accessioning.submitted.categoryId=test-pipeline-ss @@ -24,6 +25,7 @@ parameters.projectAccession=project parameters.chunkSize=5 parameters.vcf=src/test/resources/input-files/vcf/small_genotyped.vcf.gz parameters.vcfAggregation=NONE +parameters.aggregatedMappingFile=src/test/resources/output_files/aggregated_mapping_file.txt parameters.fasta=src/test/resources/input-files/fasta/Homo_sapiens.GRCh37.75.chr20.head_1200.fa parameters.outputVcf=/tmp/accession-output.vcf @@ -33,9 +35,6 @@ parameters.contigNaming=SEQUENCE_NAME spring.jpa.show-sql=true spring.data.mongodb.database=test-db -spring.data.mongodb.host=|eva.mongo.host.test| -spring.data.mongodb.password= -spring.data.mongodb.port=27017 mongodb.read-preference=primary # See https://github.com/spring-projects/spring-boot/wiki/Spring-Boot-2.1-Release-Notes#bean-overriding diff --git a/eva-accession-release/pom.xml b/eva-accession-release/pom.xml index 1c5ec5411..dfc1ce525 100644 --- a/eva-accession-release/pom.xml +++ b/eva-accession-release/pom.xml @@ -2,12 +2,13 @@ + 4.0.0 + eva-accession uk.ac.ebi.eva - 0.6.57-SNAPSHOT + ${revision} - 4.0.0 eva-accession-release jar @@ -25,7 +26,7 @@ uk.ac.ebi.eva eva-accession-core test-jar - ${project.version} + ${revision} test @@ -42,14 +43,29 @@ xstream + + org.glassfish.jaxb + jaxb-runtime + + org.springframework.batch spring-batch-test test - com.lordofthejars - nosqlunit-mongodb + org.springframework.boot + spring-boot-testcontainers + test + + + org.testcontainers + mongodb + test + + + org.testcontainers + junit-jupiter test @@ -89,7 +105,4 @@ - - localhost - diff --git a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/assembly/AssemblyNameRetriever.java b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/assembly/AssemblyNameRetriever.java index e5082d300..cbcb55138 100644 --- a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/assembly/AssemblyNameRetriever.java +++ b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/assembly/AssemblyNameRetriever.java @@ -15,11 +15,12 @@ */ package uk.ac.ebi.eva.accession.release.assembly; -import javax.xml.bind.JAXBContext; -import javax.xml.bind.JAXBException; -import javax.xml.bind.Unmarshaller; -import javax.xml.bind.annotation.XmlElement; -import javax.xml.bind.annotation.XmlRootElement; +import jakarta.xml.bind.JAXBContext; +import jakarta.xml.bind.JAXBException; +import jakarta.xml.bind.Unmarshaller; +import jakarta.xml.bind.annotation.XmlElement; +import jakarta.xml.bind.annotation.XmlRootElement; + import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; @@ -35,7 +36,7 @@ /** * Use a curated list and the ENA webservices to return the assembly name associated with an assembly accession. - * + *

* The ENA webservices for URLs like https://www.ebi.ac.uk/ena/browser/api/xml/GCA_000001405.28 * can return an xml that starts like this: * 

@@ -47,7 +48,7 @@
  * ...
  * }
  *
- * + *

* The schema for these XMLs (and the definition of the NAME element) is here: * https://github.com/enasequence/schema/blob/master/src/main/resources/uk/ac/ebi/ena/sra/schema/ENA.assembly.xsd#L44 */ @@ -96,7 +97,7 @@ private Optional fetchAssemblyName(String assemblyAccession) { String errorMessage = new BufferedReader(new InputStreamReader(httpConnection.getErrorStream())) .lines().collect(Collectors.joining("\n")); throw new RuntimeException("Unexpected response (HTTP code " + responseCode + "). Message: " - + errorMessage); + + errorMessage); } } catch (IOException | JAXBException e) { throw new RuntimeException(e); diff --git a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/VariantMongoAggregationReader.java b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/VariantMongoAggregationReader.java index d1f739cb3..a6b1c28a5 100644 --- a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/VariantMongoAggregationReader.java +++ b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/VariantMongoAggregationReader.java @@ -16,8 +16,8 @@ package uk.ac.ebi.eva.accession.release.batch.io; -import com.mongodb.MongoClient; import com.mongodb.client.AggregateIterable; +import com.mongodb.client.MongoClient; import com.mongodb.client.MongoCollection; import com.mongodb.client.MongoCursor; import com.mongodb.client.MongoDatabase; @@ -131,9 +131,8 @@ protected void aggregate(String collectionName) { logger.info("issuing aggregation on collection {}", collectionName); MongoCollection collection = db.getCollection(collectionName); AggregateIterable clusteredVariants = collection.aggregate(buildAggregation()) - .allowDiskUse(true) - .useCursor(true) - .batchSize(chunkSize); + .allowDiskUse(true) + .batchSize(chunkSize); cursor = clusteredVariants.iterator(); } @@ -151,8 +150,8 @@ protected VariantSourceEntry buildVariantSourceEntry(String study, String sequen boolean assemblyMatch, boolean evidence, boolean remappedRS, Long mergedInto) { VariantSourceEntry variantSourceEntry = buildVariantSourceEntry(study, sequenceOntology, validated, - submittedVariantValidated, allelesMatch, - assemblyMatch, evidence, remappedRS); + submittedVariantValidated, allelesMatch, + assemblyMatch, evidence, remappedRS); if (Objects.nonNull(mergedInto)) { variantSourceEntry.addAttribute(MERGED_INTO_KEY, buildId(mergedInto)); } @@ -223,15 +222,15 @@ protected boolean isSameLocation(String contig, long start, String submittedVari * The start is considered to be the same when: * - start in clustered and submitted variant match * - start in clustered and submitted variant have a difference of 1 - * + *

* The start position can be different in ambiguous INDELS because the renormalization is only applied to * submitted variants. In those cases the start in the clustered and submitted variants will not exactly match but * the difference should be 1 - * + *

* Example: * RS (assembly: GCA_000309985.1, accession: 268233057, chromosome: CM001642.1, start: 7356605, type: INS) * SS (assembly: GCA_000309985.1, accession: 490570267, chromosome: CM001642.1, start: 7356604, reference: , - * alternate: AGAGCTATGATCTTCGGAAGGAGAAGGAGAAGGAAAAGATTCATGACGTCCAC) + * alternate: AGAGCTATGATCTTCGGAAGGAGAAGGAGAAGGAAAAGATTCATGACGTCCAC) */ private boolean isSameStart(long clusteredVariantStart, long submittedVariantStart, String type) { return clusteredVariantStart == submittedVariantStart diff --git a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/active/AccessionedVariantMongoReader.java b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/active/AccessionedVariantMongoReader.java index 3151ac166..7dd435b48 100644 --- a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/active/AccessionedVariantMongoReader.java +++ b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/active/AccessionedVariantMongoReader.java @@ -16,7 +16,7 @@ package uk.ac.ebi.eva.accession.release.batch.io.active; -import com.mongodb.MongoClient; +import com.mongodb.client.MongoClient; import com.mongodb.client.model.Aggregates; import com.mongodb.client.model.Field; import com.mongodb.client.model.Filters; diff --git a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/active/VariantContextWriter.java b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/active/VariantContextWriter.java index 63116225b..13afd5f10 100644 --- a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/active/VariantContextWriter.java +++ b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/active/VariantContextWriter.java @@ -25,10 +25,10 @@ import htsjdk.variant.vcf.VCFInfoHeaderLine; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.batch.item.Chunk; import org.springframework.batch.item.ExecutionContext; import org.springframework.batch.item.ItemStreamException; import org.springframework.batch.item.ItemStreamWriter; - import uk.ac.ebi.eva.accession.release.assembly.AssemblyNameRetriever; import java.io.BufferedReader; @@ -37,7 +37,6 @@ import java.io.IOException; import java.nio.file.Path; import java.util.HashSet; -import java.util.List; import java.util.Optional; import java.util.Set; @@ -52,7 +51,7 @@ /** * Writes a VCF file for the release of RS IDs mapped against a reference assembly - * + *

* To include the contigs in the meta section it reads the file generated in the previous step * {@link ListContigsStepConfiguration} */ @@ -96,31 +95,31 @@ protected Set buildHeaderLines() { addContigs(metaData); metaData.add(new VCFHeaderLine("reference", getReferenceAssemblyLine())); metaData.add(new VCFInfoHeaderLine(VARIANT_CLASS_KEY, 1, VCFHeaderLineType.String, - "Variant class according to the Sequence Ontology")); + "Variant class according to the Sequence Ontology")); metaData.add(new VCFInfoHeaderLine(STUDY_ID_KEY, VCFHeaderLineCount.UNBOUNDED, VCFHeaderLineType.String, - "Identifiers of studies that report a variant")); + "Identifiers of studies that report a variant")); metaData.add(new VCFInfoHeaderLine(CLUSTERED_VARIANT_VALIDATED_KEY, 0, VCFHeaderLineType.Flag, - "RS validated flag, present when the RS was validated by any method " - + "as indicated by the dbSNP validation status")); + "RS validated flag, present when the RS was validated by any method " + + "as indicated by the dbSNP validation status")); metaData.add(new VCFInfoHeaderLine(SUBMITTED_VARIANT_VALIDATED_KEY, 1, VCFHeaderLineType.Integer, - "Number of submitted variants clustered in an RS that were validated by any" - + " method as indicated by the dbSNP validation status")); + "Number of submitted variants clustered in an RS that were validated by any" + + " method as indicated by the dbSNP validation status")); metaData.add(new VCFInfoHeaderLine(ALLELES_MATCH_KEY, 0, VCFHeaderLineType.Flag, - "Alleles mismatch flag, present when some of the submitted variants have " - + "inconsistent allele information. See https://github" - + ".com/EBIvariation/eva-accession/wiki/Import-accessions-from-dbSNP" - + "#alleles-match")); + "Alleles mismatch flag, present when some of the submitted variants have " + + "inconsistent allele information. See https://github" + + ".com/EBIvariation/eva-accession/wiki/Import-accessions-from-dbSNP" + + "#alleles-match")); metaData.add(new VCFInfoHeaderLine(ASSEMBLY_MATCH_KEY, 0, VCFHeaderLineType.Flag, - "Assembly mismatch flag, present when the reference allele doesn't match " - + "the reference sequence")); + "Assembly mismatch flag, present when the reference allele doesn't match " + + "the reference sequence")); metaData.add(new VCFInfoHeaderLine(SUPPORTED_BY_EVIDENCE_KEY, 0, VCFHeaderLineType.Flag, - "Lack of evidence flag, present if no submitted variant includes genotype " - + "or frequency information")); + "Lack of evidence flag, present if no submitted variant includes genotype " + + "or frequency information")); metaData.add(new VCFInfoHeaderLine(REMAPPED_KEY, 0, VCFHeaderLineType.Flag, - "Remapped flag, present if the clustered variant originate only " - + "from submitted variants remapped to this assembly")); + "Remapped flag, present if the clustered variant originate only " + + "from submitted variants remapped to this assembly")); return metaData; } @@ -152,7 +151,7 @@ private void addContigs(Set metaData) { } @Override - public void write(List variantContexts) throws Exception { + public void write(Chunk variantContexts) throws Exception { for (VariantContext variantContext : variantContexts) { writer.add(variantContext); } diff --git a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/contig/ContigMongoReader.java b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/contig/ContigMongoReader.java index 4c8487664..4cae6d04f 100644 --- a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/contig/ContigMongoReader.java +++ b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/contig/ContigMongoReader.java @@ -16,8 +16,8 @@ package uk.ac.ebi.eva.accession.release.batch.io.contig; -import com.mongodb.MongoClient; import com.mongodb.client.AggregateIterable; +import com.mongodb.client.MongoClient; import com.mongodb.client.MongoCollection; import com.mongodb.client.MongoCursor; import com.mongodb.client.MongoDatabase; @@ -34,14 +34,13 @@ import org.springframework.batch.item.ParseException; import org.springframework.batch.item.UnexpectedInputException; import uk.ac.ebi.ampt2d.commons.accession.core.models.EventType; - import uk.ac.ebi.eva.accession.release.collectionNames.CollectionNames; import java.util.Arrays; import java.util.List; -import static uk.ac.ebi.eva.accession.release.batch.io.multimap.MultimapVariantMongoReader.NON_SINGLE_LOCATION_MAPPING; import static uk.ac.ebi.eva.accession.release.batch.io.multimap.MultimapVariantMongoReader.MAPPING_WEIGHT_FIELD; +import static uk.ac.ebi.eva.accession.release.batch.io.multimap.MultimapVariantMongoReader.NON_SINGLE_LOCATION_MAPPING; public class ContigMongoReader implements ItemStreamReader { @@ -83,22 +82,22 @@ public static ContigMongoReader activeContigReader(String assemblyAccession, int MongoClient mongoClient, String database, CollectionNames names) { return new ContigMongoReader(assemblyAccession, mongoClient, database, - names.getSubmittedVariantEntity(), - buildAggregationForActiveContigs(assemblyAccession, taxonomyAccession)); + names.getSubmittedVariantEntity(), + buildAggregationForActiveContigs(assemblyAccession, taxonomyAccession)); } public static ContigMongoReader mergedContigReader(String assemblyAccession, MongoClient mongoClient, String database, CollectionNames names) { return new ContigMongoReader(assemblyAccession, mongoClient, database, - names.getClusteredVariantOperationEntity(), - buildAggregationForMergedContigs(assemblyAccession)); + names.getClusteredVariantOperationEntity(), + buildAggregationForMergedContigs(assemblyAccession)); } public static ContigMongoReader multimapContigReader(String assemblyAccession, MongoClient mongoClient, String database, CollectionNames names) { return new ContigMongoReader(assemblyAccession, mongoClient, database, - names.getClusteredVariantEntity(), - buildAggregationForMultimapContigs(assemblyAccession)); + names.getClusteredVariantEntity(), + buildAggregationForMultimapContigs(assemblyAccession)); } private ContigMongoReader(String assemblyAccession, MongoClient mongoClient, String database, String collection, @@ -122,20 +121,20 @@ private static List buildAggregationForActiveContigs(String assemblyAccess /** * Apparently, a $group aggregation stage yields a different result * if you do the $group on a nested field like "inactiveObjects.contig": - * + *

* { "_id" : [ "KB882311.1" ] } - * + *

* while if the $group is done on a simple toplevel field it returns - * + *

* { "_id" : "KB882311.1" } - * + *

* so, as a $group on an array is not what we want to do, we have to * $project the contig field into a toplevel field * before we can do the $group. */ private static List buildAggregationForMergedContigs(String assemblyAccession) { Bson match = Aggregates.match(Filters.and(Filters.eq(INACTIVE_REFERENCE_ASSEMBLY_FIELD, assemblyAccession), - Filters.eq(EVENT_TYPE_FIELD, EventType.MERGED.toString()))); + Filters.eq(EVENT_TYPE_FIELD, EventType.MERGED.toString()))); Bson extractContig = Aggregates.project(new Document(MONGO_ID_FIELD, INACTIVE_CONTIG_KEY)); @@ -151,7 +150,7 @@ private static List buildAggregationForMergedContigs(String assemblyAccess private static List buildAggregationForMultimapContigs(String assemblyAccession) { Bson match = Aggregates.match(Filters.and(Filters.eq(ACTIVE_REFERENCE_ASSEMBLY_FIELD, assemblyAccession), - Filters.gte(MAPPING_WEIGHT_FIELD, NON_SINGLE_LOCATION_MAPPING))); + Filters.gte(MAPPING_WEIGHT_FIELD, NON_SINGLE_LOCATION_MAPPING))); Bson uniqueContigs = Aggregates.group(ACTIVE_CONTIG_KEY); List aggregation = Arrays.asList(match, uniqueContigs); logger.info("Issuing aggregation: {}", aggregation); @@ -168,8 +167,7 @@ private void aggregate() { MongoDatabase db = mongoClient.getDatabase(database); MongoCollection mongoCollection = db.getCollection(collection); AggregateIterable clusteredVariants = mongoCollection.aggregate(aggregation) - .allowDiskUse(true) - .useCursor(true); + .allowDiskUse(true); cursor = clusteredVariants.iterator(); } diff --git a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/contig/ContigWriter.java b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/contig/ContigWriter.java index 853cee24d..a676030b8 100644 --- a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/contig/ContigWriter.java +++ b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/contig/ContigWriter.java @@ -15,10 +15,10 @@ */ package uk.ac.ebi.eva.accession.release.batch.io.contig; +import org.springframework.batch.item.Chunk; import org.springframework.batch.item.ExecutionContext; import org.springframework.batch.item.ItemStreamException; import org.springframework.batch.item.ItemStreamWriter; - import uk.ac.ebi.eva.accession.core.contig.ContigMapping; import uk.ac.ebi.eva.accession.core.contig.ContigNaming; import uk.ac.ebi.eva.accession.core.contig.ContigSynonyms; @@ -28,7 +28,6 @@ import java.io.IOException; import java.io.PrintWriter; import java.nio.file.Paths; -import java.util.List; /** * Writes the list of contigs to a flat file @@ -78,7 +77,7 @@ public void close() throws ItemStreamException { } @Override - public void write(List contigs) { + public void write(Chunk contigs) { for (String contig : contigs) { if (contig == null || contig.isEmpty()) { throw new IllegalArgumentException("The contig cannot be null or empty"); diff --git a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/deprecated/DeprecatedVariantAccessionWriter.java b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/deprecated/DeprecatedVariantAccessionWriter.java index 3e3362dba..2d4666268 100644 --- a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/deprecated/DeprecatedVariantAccessionWriter.java +++ b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/deprecated/DeprecatedVariantAccessionWriter.java @@ -15,10 +15,10 @@ */ package uk.ac.ebi.eva.accession.release.batch.io.deprecated; +import org.springframework.batch.item.Chunk; import org.springframework.batch.item.ExecutionContext; import org.springframework.batch.item.ItemStreamException; import org.springframework.batch.item.ItemStreamWriter; - import uk.ac.ebi.eva.commons.core.models.pipeline.Variant; import java.io.File; @@ -26,7 +26,6 @@ import java.io.IOException; import java.io.PrintWriter; import java.nio.file.Path; -import java.util.List; import java.util.Objects; /** @@ -66,8 +65,8 @@ public void close() throws ItemStreamException { } @Override - public void write(List variants) throws Exception { - for (Variant variant: variants) { + public void write(Chunk variants) throws Exception { + for (Variant variant : variants) { if (!Objects.isNull(variant)) { printWriter.println(variant.getMainId()); } diff --git a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/deprecated/DeprecatedVariantMongoReader.java b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/deprecated/DeprecatedVariantMongoReader.java index 041ce166f..eb9380a34 100644 --- a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/deprecated/DeprecatedVariantMongoReader.java +++ b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/deprecated/DeprecatedVariantMongoReader.java @@ -15,7 +15,7 @@ */ package uk.ac.ebi.eva.accession.release.batch.io.deprecated; -import com.mongodb.MongoClient; +import com.mongodb.client.MongoClient; import com.mongodb.client.model.Aggregates; import com.mongodb.client.model.Field; import com.mongodb.client.model.Filters; @@ -67,33 +67,33 @@ private String getInactiveField(String field) { @Override protected List buildAggregation() { Bson matchAssembly = Aggregates.match(Filters.eq(getInactiveField(REFERENCE_ASSEMBLY_FIELD), - assemblyAccession)); + assemblyAccession)); Bson matchMerged = Aggregates.match(Filters.eq(EVENT_TYPE_FIELD, EventType.DEPRECATED.toString())); Bson sort = Aggregates.sort(orderBy(ascending(getInactiveField(CONTIG_FIELD), getInactiveField(START_FIELD)))); List aggregation = new ArrayList<>(Arrays.asList(matchAssembly, matchMerged, sort)); for (String submittedVariantOperationCollectionName : allSubmittedVariantOperationCollectionNames) { Bson lookup = Aggregates.lookup(submittedVariantOperationCollectionName, ACCESSION_FIELD, - getInactiveField(CLUSTERED_VARIANT_ACCESSION_FIELD), - submittedVariantOperationCollectionName); + getInactiveField(CLUSTERED_VARIANT_ACCESSION_FIELD), + submittedVariantOperationCollectionName); aggregation.add(lookup); } // Concat entries from all submitted variant operation collections Bson ssConcat = Aggregates.addFields(new Field<>(SS_INFO_FIELD, - new Document("$concatArrays", allSubmittedVariantOperationCollectionNames - .stream().map(v -> "$" + v) - .collect(Collectors.toList())))); + new Document("$concatArrays", allSubmittedVariantOperationCollectionNames + .stream().map(v -> "$" + v) + .collect(Collectors.toList())))); aggregation.add(ssConcat); // Ensure that we are only retrieving the variants with the relevant taxonomy // and event type in the Submitted operations collections Bson matchTaxonomyAndEventType = Aggregates.match(Filters.and( Filters.ne(SS_INFO_FIELD, Collections.emptyList()), Filters.eq(SS_INFO_FIELD + "." + - getInactiveField(REFERENCE_ASSEMBLY_FIELD_IN_SUBMITTED_COLLECTIONS), - this.assemblyAccession), + getInactiveField(REFERENCE_ASSEMBLY_FIELD_IN_SUBMITTED_COLLECTIONS), + this.assemblyAccession), Filters.eq(SS_INFO_FIELD + "." + getInactiveField(TAXONOMY_FIELD), this.taxonomyAccession), Filters.in(SS_INFO_FIELD + "." + EVENT_TYPE_FIELD, - Arrays.asList(EventType.UPDATED.toString(), EventType.DEPRECATED.toString())))); + Arrays.asList(EventType.UPDATED.toString(), EventType.DEPRECATED.toString())))); aggregation.add(matchTaxonomyAndEventType); logger.info("Issuing aggregation: {}", aggregation); logger.info(aggregation.toString()); diff --git a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/merged/MergedVariantContextWriter.java b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/merged/MergedVariantContextWriter.java index 93712ab4b..fbbf2d77d 100644 --- a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/merged/MergedVariantContextWriter.java +++ b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/merged/MergedVariantContextWriter.java @@ -18,9 +18,8 @@ import htsjdk.variant.vcf.VCFHeaderLine; import htsjdk.variant.vcf.VCFHeaderLineType; import htsjdk.variant.vcf.VCFInfoHeaderLine; - -import uk.ac.ebi.eva.accession.release.batch.io.active.VariantContextWriter; import uk.ac.ebi.eva.accession.release.batch.io.VariantMongoAggregationReader; +import uk.ac.ebi.eva.accession.release.batch.io.active.VariantContextWriter; import java.nio.file.Path; import java.util.Set; diff --git a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/merged/MergedVariantMongoReader.java b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/merged/MergedVariantMongoReader.java index 6a43bdcc0..8daa07389 100644 --- a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/merged/MergedVariantMongoReader.java +++ b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/merged/MergedVariantMongoReader.java @@ -16,7 +16,7 @@ package uk.ac.ebi.eva.accession.release.batch.io.merged; -import com.mongodb.MongoClient; +import com.mongodb.client.MongoClient; import com.mongodb.client.model.Aggregates; import com.mongodb.client.model.Field; import com.mongodb.client.model.Filters; @@ -94,44 +94,44 @@ public void open(ExecutionContext executionContext) throws ItemStreamException { @Override protected List buildAggregation() { Bson matchAssembly = Aggregates.match(Filters.eq(getInactiveField(REFERENCE_ASSEMBLY_FIELD), - assemblyAccession)); + assemblyAccession)); Bson matchMerged = Aggregates.match(Filters.eq(EVENT_TYPE_FIELD, EventType.MERGED.toString())); Bson sort = Aggregates.sort(orderBy(ascending(getInactiveField(CONTIG_FIELD), getInactiveField(START_FIELD)))); List aggregation = new ArrayList<>(Arrays.asList(matchAssembly, matchMerged, sort)); for (String submittedVariantOperationCollectionName : allSubmittedVariantOperationCollectionNames) { Bson lookup = Aggregates.lookup(submittedVariantOperationCollectionName, ACCESSION_FIELD, - getInactiveField(CLUSTERED_VARIANT_ACCESSION_FIELD), - submittedVariantOperationCollectionName); + getInactiveField(CLUSTERED_VARIANT_ACCESSION_FIELD), + submittedVariantOperationCollectionName); aggregation.add(lookup); } // Concat entries from all submitted variant operation collections Bson ssConcat = Aggregates.addFields(new Field<>(SS_INFO_FIELD, - new Document("$concatArrays", allSubmittedVariantOperationCollectionNames - .stream().map(v -> "$" + v) - .collect(Collectors.toList())))); + new Document("$concatArrays", allSubmittedVariantOperationCollectionNames + .stream().map(v -> "$" + v) + .collect(Collectors.toList())))); aggregation.add(ssConcat); // Ensure that we are only retrieving the variants with the relevant taxonomy // and event type in the Submitted operations collections Bson matchTaxonomyAndEventType = Aggregates.match(Filters.and( Filters.ne(SS_INFO_FIELD, Collections.emptyList()), Filters.eq(SS_INFO_FIELD + "." + - getInactiveField(REFERENCE_ASSEMBLY_FIELD_IN_SUBMITTED_COLLECTIONS), - this.assemblyAccession), + getInactiveField(REFERENCE_ASSEMBLY_FIELD_IN_SUBMITTED_COLLECTIONS), + this.assemblyAccession), Filters.eq(SS_INFO_FIELD + "." + getInactiveField(TAXONOMY_FIELD), this.taxonomyAccession), - Filters.eq(SS_INFO_FIELD + "." + EVENT_TYPE_FIELD,EventType.UPDATED.toString()))); + Filters.eq(SS_INFO_FIELD + "." + EVENT_TYPE_FIELD, EventType.UPDATED.toString()))); aggregation.add(matchTaxonomyAndEventType); // Similarly look in both clustered variant collections for active RS for (String clusteredVariantCollectionName : allClusteredVariantCollectionNames) { Bson lookupClusteredVariants = Aggregates.lookup(clusteredVariantCollectionName, MERGE_INTO_FIELD, - ACCESSION_FIELD, clusteredVariantCollectionName); + ACCESSION_FIELD, clusteredVariantCollectionName); aggregation.add(lookupClusteredVariants); } Bson rsConcat = Aggregates.addFields(new Field<>(ACTIVE_RS, - new Document("$concatArrays", allClusteredVariantCollectionNames - .stream().map(v -> "$" + v) - .collect(Collectors.toList())))); + new Document("$concatArrays", allClusteredVariantCollectionNames + .stream().map(v -> "$" + v) + .collect(Collectors.toList())))); aggregation.add(rsConcat); Bson matchOnlyNonEmptyActiveRS = Aggregates.match(Filters.ne(ACTIVE_RS, Collections.emptyList())); aggregation.add(matchOnlyNonEmptyActiveRS); @@ -148,9 +148,9 @@ protected List getVariants(Document mergedVariant) { Collection inactiveObjects = (Collection) mergedVariant.get(INACTIVE_OBJECTS); if (inactiveObjects.size() > 1) { throw new AssertionError("The class '" + this.getClass().getSimpleName() - + "' was designed assuming there's only one element in the field " - + "'" + INACTIVE_OBJECTS + "'. Found " + inactiveObjects.size() - + " elements in _id=" + mergedVariant.get(ACCESSION_FIELD)); + + "' was designed assuming there's only one element in the field " + + "'" + INACTIVE_OBJECTS + "'. Found " + inactiveObjects.size() + + " elements in _id=" + mergedVariant.get(ACCESSION_FIELD)); } Document inactiveEntity = inactiveObjects.iterator().next(); String contig = inactiveEntity.getString(VariantMongoAggregationReader.CONTIG_FIELD); @@ -164,9 +164,9 @@ protected List getVariants(Document mergedVariant) { Map mergedVariants = new HashMap<>(); Collection submittedVariantOperations = (Collection) mergedVariant.get(SS_INFO_FIELD); boolean remappedRS = submittedVariantOperations.stream() - .map(e -> (Collection) e.get("inactiveObjects")) - .flatMap(Collection::stream) - .allMatch(sve -> Objects.nonNull(sve.getString("remappedFrom"))); + .map(e -> (Collection) e.get("inactiveObjects")) + .flatMap(Collection::stream) + .allMatch(sve -> Objects.nonNull(sve.getString("remappedFrom"))); Collection activeClusteredVariant = (Collection) mergedVariant.get(ACTIVE_RS); if (activeClusteredVariant.isEmpty()) { @@ -183,7 +183,7 @@ protected List getVariants(Document mergedVariant) { long submittedVariantStart = submittedVariant.getLong(START_FIELD); String submittedVariantContig = submittedVariant.getString(CONTIG_FIELD); - if (!isSameLocation(contig, start, submittedVariantContig, submittedVariantStart, type)){ + if (!isSameLocation(contig, start, submittedVariantContig, submittedVariantStart, type)) { continue; } @@ -197,9 +197,9 @@ protected List getVariants(Document mergedVariant) { .getBoolean(SUPPORTED_BY_EVIDENCE_FIELD, DEFAULT_SUPPORTED_BY_EVIDENCE); VariantSourceEntry sourceEntry = buildVariantSourceEntry(study, sequenceOntology, validated, - submittedVariantValidated, allelesMatch, - assemblyMatch, evidence, remappedRS, - mergedInto); + submittedVariantValidated, allelesMatch, + assemblyMatch, evidence, remappedRS, + mergedInto); addToVariants(mergedVariants, contig, submittedVariantStart, rs, reference, alternate, sourceEntry); } @@ -212,10 +212,10 @@ protected List getVariants(Document mergedVariant) { if (!hasSubmittedVariantsDeclustered && mergedVariants.isEmpty()) { logger.warn("Found merge operation for rs" + rs + " but no SS IDs updates " + - "(merge/update) in the collection containing operations. " + - "This could have possibly happened on a remapped variant " + - "because there was a subsequent split issued for this RS due to loci disagreement " + - "between the RS and the SS."); + "(merge/update) in the collection containing operations. " + + "This could have possibly happened on a remapped variant " + + "because there was a subsequent split issued for this RS due to loci disagreement " + + "between the RS and the SS."); return Collections.emptyList(); } diff --git a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/merged_deprecated/DbsnpMergedDeprecatedVariantMongoReader.java b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/merged_deprecated/DbsnpMergedDeprecatedVariantMongoReader.java index 071ab00ab..ffd885b98 100644 --- a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/merged_deprecated/DbsnpMergedDeprecatedVariantMongoReader.java +++ b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/merged_deprecated/DbsnpMergedDeprecatedVariantMongoReader.java @@ -17,7 +17,7 @@ package uk.ac.ebi.eva.accession.release.batch.io.merged_deprecated; import com.mongodb.BasicDBObject; -import com.mongodb.MongoClient; +import com.mongodb.client.MongoClient; import org.bson.Document; import org.springframework.data.mongodb.core.convert.MongoConverter; @@ -55,10 +55,10 @@ public DbsnpMergedDeprecatedVariantMongoReader(String assemblyAccession, int tax protected DbsnpClusteredVariantOperationEntity getEntity(Document operation) { List objects = (List) operation.get(INACTIVE_OBJECTS); operation.put(INACTIVE_OBJECTS, objects.stream() - .map(BasicDBObject::new) - .map(o -> mongoConverter.read(DbsnpClusteredVariantInactiveEntity.class, - o)) - .collect(Collectors.toList())); + .map(BasicDBObject::new) + .map(o -> mongoConverter.read(DbsnpClusteredVariantInactiveEntity.class, + o)) + .collect(Collectors.toList())); return mongoConverter.read(DbsnpClusteredVariantOperationEntity.class, new BasicDBObject(operation)); } diff --git a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/merged_deprecated/EvaMergedDeprecatedVariantMongoReader.java b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/merged_deprecated/EvaMergedDeprecatedVariantMongoReader.java index 8fc151ad4..de078399c 100644 --- a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/merged_deprecated/EvaMergedDeprecatedVariantMongoReader.java +++ b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/merged_deprecated/EvaMergedDeprecatedVariantMongoReader.java @@ -17,7 +17,7 @@ package uk.ac.ebi.eva.accession.release.batch.io.merged_deprecated; import com.mongodb.BasicDBObject; -import com.mongodb.MongoClient; +import com.mongodb.client.MongoClient; import org.bson.Document; import org.springframework.data.mongodb.core.convert.MongoConverter; import uk.ac.ebi.ampt2d.commons.accession.persistence.mongodb.document.EventDocument; @@ -56,10 +56,10 @@ public EvaMergedDeprecatedVariantMongoReader(String assemblyAccession, int taxon protected ClusteredVariantOperationEntity getEntity(Document operation) { List objects = (List) operation.get(INACTIVE_OBJECTS); operation.put(INACTIVE_OBJECTS, objects.stream() - .map(BasicDBObject::new) - .map(o -> mongoConverter.read(ClusteredVariantInactiveEntity.class, - o)) - .collect(Collectors.toList())); + .map(BasicDBObject::new) + .map(o -> mongoConverter.read(ClusteredVariantInactiveEntity.class, + o)) + .collect(Collectors.toList())); return mongoConverter.read(ClusteredVariantOperationEntity.class, new BasicDBObject(operation)); } diff --git a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/merged_deprecated/MergedDeprecatedVariantAccessionWriter.java b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/merged_deprecated/MergedDeprecatedVariantAccessionWriter.java index 3e6a9ce06..4577ce040 100644 --- a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/merged_deprecated/MergedDeprecatedVariantAccessionWriter.java +++ b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/merged_deprecated/MergedDeprecatedVariantAccessionWriter.java @@ -15,6 +15,7 @@ */ package uk.ac.ebi.eva.accession.release.batch.io.merged_deprecated; +import org.springframework.batch.item.Chunk; import org.springframework.batch.item.ExecutionContext; import org.springframework.batch.item.ItemStreamException; import org.springframework.batch.item.ItemStreamWriter; @@ -27,7 +28,6 @@ import java.io.IOException; import java.io.PrintWriter; import java.nio.file.Path; -import java.util.List; /** * Writes the accessions of historical variants, that have been merged into a later deprecate one, to a flat file. @@ -57,7 +57,7 @@ public void open(ExecutionContext executionContext) throws ItemStreamException { } @Override - public void write(List>> variants) throws Exception { for (EventDocument> variant : variants) { diff --git a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/merged_deprecated/MergedDeprecatedVariantMongoReader.java b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/merged_deprecated/MergedDeprecatedVariantMongoReader.java index db5522712..270941a31 100644 --- a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/merged_deprecated/MergedDeprecatedVariantMongoReader.java +++ b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/merged_deprecated/MergedDeprecatedVariantMongoReader.java @@ -16,8 +16,8 @@ package uk.ac.ebi.eva.accession.release.batch.io.merged_deprecated; -import com.mongodb.MongoClient; import com.mongodb.client.AggregateIterable; +import com.mongodb.client.MongoClient; import com.mongodb.client.MongoCollection; import com.mongodb.client.MongoCursor; import com.mongodb.client.MongoDatabase; @@ -31,11 +31,7 @@ import org.springframework.batch.item.ExecutionContext; import org.springframework.batch.item.ItemStreamException; import org.springframework.batch.item.ItemStreamReader; -import org.springframework.batch.item.NonTransientResourceException; -import org.springframework.batch.item.ParseException; -import org.springframework.batch.item.UnexpectedInputException; import uk.ac.ebi.ampt2d.commons.accession.core.models.EventType; - import uk.ac.ebi.eva.accession.release.collectionNames.CollectionNames; import java.util.ArrayList; @@ -110,9 +106,8 @@ private void aggregate(String collectionName) { MongoDatabase db = mongoClient.getDatabase(database); MongoCollection collection = db.getCollection(collectionName); AggregateIterable clusteredVariants = collection.aggregate(buildAggregation()) - .allowDiskUse(true) - .useCursor(true) - .batchSize(chunkSize); + .allowDiskUse(true) + .batchSize(chunkSize); cursor = clusteredVariants.iterator(); } @@ -121,32 +116,32 @@ private List buildAggregation() { Bson matchAssembly = Aggregates.match(Filters.eq(getInactiveField(ASSEMBLY_FIELD), assemblyAccession)); Bson matchMerged = Aggregates.match(Filters.eq(EVENT_TYPE_FIELD, EventType.MERGED.toString())); Bson lookup = Aggregates.lookup(names.getSubmittedVariantEntity(), MERGE_INTO_FIELD, - CLUSTERED_VARIANT_ACCESSION_FIELD, SS_INFO_FIELD); + CLUSTERED_VARIANT_ACCESSION_FIELD, SS_INFO_FIELD); Bson matchEmpty = Aggregates.match(Filters.eq(SS_INFO_FIELD, Collections.EMPTY_LIST)); aggregation.addAll(Arrays.asList(matchAssembly, matchMerged, lookup, matchEmpty)); for (String submittedVariantOperationCollectionName : allSubmittedVariantOperationCollectionNames) { Bson opsLookup = Aggregates.lookup(submittedVariantOperationCollectionName, ACCESSION_FIELD, - getInactiveField(CLUSTERED_VARIANT_ACCESSION_FIELD), - submittedVariantOperationCollectionName); + getInactiveField(CLUSTERED_VARIANT_ACCESSION_FIELD), + submittedVariantOperationCollectionName); aggregation.add(opsLookup); } // Concat entries from all submitted variant operation collections Bson ssOpsConcat = Aggregates.addFields(new Field<>(SS_OPS_INFO_FIELD, - new Document("$concatArrays", - allSubmittedVariantOperationCollectionNames - .stream().map(v -> "$" + v) - .collect(Collectors.toList())))); + new Document("$concatArrays", + allSubmittedVariantOperationCollectionNames + .stream().map(v -> "$" + v) + .collect(Collectors.toList())))); aggregation.add(ssOpsConcat); // There should be some evidence that there is at least one operation in the submitted operations // referencing the said RS Bson matchAtLeastOneSSOp = Aggregates.match(Filters.and( Filters.ne(SS_OPS_INFO_FIELD, Collections.emptyList()), Filters.eq(SS_OPS_INFO_FIELD + "." + - getInactiveField(TAXONOMY_FIELD), - this.taxonomyAccession), + getInactiveField(TAXONOMY_FIELD), + this.taxonomyAccession), Filters.eq(SS_OPS_INFO_FIELD + "." + - getInactiveField(REFERENCE_ASSEMBLY_FIELD_IN_SUBMITTED_COLLECTIONS), - this.assemblyAccession))); + getInactiveField(REFERENCE_ASSEMBLY_FIELD_IN_SUBMITTED_COLLECTIONS), + this.assemblyAccession))); aggregation.add(matchAtLeastOneSSOp); logger.info("Issuing aggregation: {}", aggregation); return aggregation; @@ -157,8 +152,7 @@ private String getInactiveField(String field) { } @Override - public OPERATION_ENTITY read() - throws Exception, UnexpectedInputException, ParseException, NonTransientResourceException { + public OPERATION_ENTITY read() throws Exception { return cursor.hasNext() ? getEntity(cursor.next()) : null; } diff --git a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/multimap/MultimapVariantContextWriter.java b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/multimap/MultimapVariantContextWriter.java index 9693ab369..19ab2bd30 100644 --- a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/multimap/MultimapVariantContextWriter.java +++ b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/multimap/MultimapVariantContextWriter.java @@ -39,8 +39,8 @@ public MultimapVariantContextWriter(Path outputPath, String referenceAssembly, S protected Set buildHeaderLines() { Set vcfHeaderLines = super.buildHeaderLines(); vcfHeaderLines.add(new VCFInfoHeaderLine(MAPPING_WEIGHT_KEY, 1, VCFHeaderLineType.Integer, - "mapping weight as defined by dbSNP for database tables at " + - DBSNP_MAP_WEIGHT_DEFINITION_URL)); + "mapping weight as defined by dbSNP for database tables at " + + DBSNP_MAP_WEIGHT_DEFINITION_URL)); return vcfHeaderLines; } diff --git a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/multimap/MultimapVariantMongoReader.java b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/multimap/MultimapVariantMongoReader.java index a7f7446f1..647df8433 100644 --- a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/multimap/MultimapVariantMongoReader.java +++ b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/multimap/MultimapVariantMongoReader.java @@ -15,7 +15,7 @@ */ package uk.ac.ebi.eva.accession.release.batch.io.multimap; -import com.mongodb.MongoClient; +import com.mongodb.client.MongoClient; import com.mongodb.client.model.Aggregates; import com.mongodb.client.model.Filters; import org.bson.Document; @@ -48,11 +48,11 @@ public MultimapVariantMongoReader(String assemblyAccession, int taxonomyAccessio @Override protected List buildAggregation() { Bson match = Aggregates.match(Filters.and(Filters.eq(REFERENCE_ASSEMBLY_FIELD, assemblyAccession), - Filters.eq(TAXONOMY_FIELD, taxonomyAccession), - Filters.gte(MAPPING_WEIGHT_FIELD, NON_SINGLE_LOCATION_MAPPING))); + Filters.eq(TAXONOMY_FIELD, taxonomyAccession), + Filters.gte(MAPPING_WEIGHT_FIELD, NON_SINGLE_LOCATION_MAPPING))); Bson sort = Aggregates.sort(orderBy(ascending(CONTIG_FIELD, START_FIELD))); Bson lookup = Aggregates.lookup(names.getSubmittedVariantEntity(), ACCESSION_FIELD, - CLUSTERED_VARIANT_ACCESSION_FIELD, SS_INFO_FIELD); + CLUSTERED_VARIANT_ACCESSION_FIELD, SS_INFO_FIELD); List aggregation = Arrays.asList(match, sort, lookup); logger.info("Issuing aggregation: {}", aggregation); return aggregation; @@ -63,7 +63,7 @@ protected List getVariants(Document clusteredVariant) { List variants = super.getVariants(clusteredVariant); for (Variant variant : variants) { variant.getSourceEntries().iterator().next().addAttribute(MAPPING_WEIGHT_KEY, - clusteredVariant.get(MAPPING_WEIGHT_FIELD).toString()); + clusteredVariant.get(MAPPING_WEIGHT_FIELD).toString()); } return variants; } diff --git a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/listeners/ExcludeVariantsListener.java b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/listeners/ExcludeVariantsListener.java index 075e590cb..3446ac767 100644 --- a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/listeners/ExcludeVariantsListener.java +++ b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/listeners/ExcludeVariantsListener.java @@ -32,7 +32,7 @@ public class ExcludeVariantsListener extends StepListenerSupport * VCF format only accepts reference and alternate alleles formed by A, C, G, T or N letters in upper or lower case. * If one of the alleles has a different character, this processor will return null so that variant can be ignored. */ @@ -40,9 +40,6 @@ public class ExcludeInvalidVariantsProcessor implements ItemProcessor * Examples of structural variants taken from * VCFv.3 spec (section 5.3): * 

@@ -47,9 +47,9 @@
  * 3       9425916 .    C      23   PASS   SVTYPE=INS;END=9425916;SVLEN=6027
  * }
  *
- * + *

* Note that unlike regular INDELS, variants with symbolic alleles have the context bases only in the REF column. - * + *

* Also, note that the deletions have the symbolic allele in the ALT column. */ public class NamedVariantProcessor implements ItemProcessor { @@ -58,9 +58,9 @@ public class NamedVariantProcessor implements ItemProcessor { private static Map sequenceOntologyToVariantType = Arrays.stream(VariantType.values()) - .filter(type -> type != VariantType.NO_ALTERNATE) - .collect(Collectors.toMap(VariantTypeToSOAccessionMap::getSequenceOntologyAccession, - type -> type)); + .filter(type -> type != VariantType.NO_ALTERNATE) + .collect(Collectors.toMap(VariantTypeToSOAccessionMap::getSequenceOntologyAccession, + type -> type)); @Override public IVariant process(Variant variant) throws Exception { @@ -75,7 +75,7 @@ public IVariant process(Variant variant) throws Exception { if (isNamedAllele(oldAlternate) || isSymbolicAllele(oldAlternate)) { throw new IllegalArgumentException( "This variant (with named/symbolic alleles in both the reference and alternate alleles) can't" - + " be written in VCF, as only the ALT column can have symbolic alleles: " + variant); + + " be written in VCF, as only the ALT column can have symbolic alleles: " + variant); } else { // swap the alleles, look this class' documentation newReference = oldAlternate; @@ -86,7 +86,7 @@ public IVariant process(Variant variant) throws Exception { } Variant newVariant = new Variant(variant.getChromosome(), variant.getStart(), variant.getEnd(), newReference, - makeAltAlleleValid(newAlternate)); + makeAltAlleleValid(newAlternate)); newVariant.addSourceEntries(variant.getSourceEntries()); newVariant.setMainId(variant.getMainId()); @@ -131,19 +131,19 @@ private void logIfVariantHasWrongType(Variant variant) { boolean isSequenceAlteration = variant.getType() == VariantType.SEQUENCE_ALTERATION; if (isSequenceAlteration && !containsSequenceAlterationInMongo - || !isSequenceAlteration && containsSequenceAlterationInMongo) { + || !isSequenceAlteration && containsSequenceAlterationInMongo) { logger.warn("Variant is stored in MongoDB with types " + types + ", but AbstractVariant::getType says" - + " that the correct type is " + variant.getType() + ". The variant (" + variant.getMainId() - + ") is: " + variant); + + " that the correct type is " + variant.getType() + ". The variant (" + variant.getMainId() + + ") is: " + variant); } } private Set getTypeFromAttributes(Variant variant) { return variant.getSourceEntries() - .stream() - .map(AbstractVariantSourceEntry::getAttributes) - .map(attributes -> attributes.get(VARIANT_CLASS_KEY)) - .map(sequenceOntologyToVariantType::get) - .collect(Collectors.toSet()); + .stream() + .map(AbstractVariantSourceEntry::getAttributes) + .map(attributes -> attributes.get(VARIANT_CLASS_KEY)) + .map(sequenceOntologyToVariantType::get) + .collect(Collectors.toSet()); } } diff --git a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/processors/VariantToVariantContextProcessor.java b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/processors/VariantToVariantContextProcessor.java index 4d3e409a5..6b4803048 100644 --- a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/processors/VariantToVariantContextProcessor.java +++ b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/processors/VariantToVariantContextProcessor.java @@ -19,7 +19,6 @@ import htsjdk.variant.variantcontext.VariantContext; import htsjdk.variant.variantcontext.VariantContextBuilder; import org.springframework.batch.item.ItemProcessor; - import uk.ac.ebi.eva.accession.core.contig.ContigMapping; import uk.ac.ebi.eva.accession.core.contig.ContigNaming; import uk.ac.ebi.eva.accession.core.contig.ContigSynonyms; @@ -33,6 +32,7 @@ import java.util.Map; import java.util.stream.Collectors; +import static uk.ac.ebi.eva.accession.release.batch.io.VariantMongoAggregationReader.MAPPING_WEIGHT_KEY; import static uk.ac.ebi.eva.accession.release.batch.io.active.AccessionedVariantMongoReader.ALLELES_MATCH_KEY; import static uk.ac.ebi.eva.accession.release.batch.io.active.AccessionedVariantMongoReader.ASSEMBLY_MATCH_KEY; import static uk.ac.ebi.eva.accession.release.batch.io.active.AccessionedVariantMongoReader.CLUSTERED_VARIANT_VALIDATED_KEY; @@ -42,11 +42,10 @@ import static uk.ac.ebi.eva.accession.release.batch.io.active.AccessionedVariantMongoReader.SUPPORTED_BY_EVIDENCE_KEY; import static uk.ac.ebi.eva.accession.release.batch.io.active.AccessionedVariantMongoReader.VARIANT_CLASS_KEY; import static uk.ac.ebi.eva.accession.release.batch.io.merged.MergedVariantMongoReader.MERGED_INTO_KEY; -import static uk.ac.ebi.eva.accession.release.batch.io.VariantMongoAggregationReader.MAPPING_WEIGHT_KEY; /** * Converts an IVariant to a VariantContext. - * + *

* The latter can be serialized using HTSJDK. This processor requires any to-be-serialized property to be set in the * IVariant already, such as the alleles. */ @@ -103,7 +102,7 @@ private Map getAttributes(IVariant variant) { case MERGED_INTO_KEY: case MAPPING_WEIGHT_KEY: attributes.put(attribute.getKey(), - toUniqueConcatenation(replaceInvalidCharacters(attribute.getValue()))); + toUniqueConcatenation(replaceInvalidCharacters(attribute.getValue()))); break; case ALLELES_MATCH_KEY: case ASSEMBLY_MATCH_KEY: @@ -141,8 +140,8 @@ private Map getAttributes(IVariant variant) { */ private List replaceInvalidCharacters(List infoValues) { return infoValues.stream() - .map(s -> s.replaceAll("[ ,;=]", "_")) - .collect(Collectors.toList()); + .map(s -> s.replaceAll("[ ,;=]", "_")) + .collect(Collectors.toList()); } private String toUniqueConcatenation(List value) { diff --git a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/RunnerConfiguration.java b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/RunnerConfiguration.java index a9792312b..a0c05b893 100644 --- a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/RunnerConfiguration.java +++ b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/RunnerConfiguration.java @@ -16,12 +16,6 @@ */ package uk.ac.ebi.eva.accession.release.configuration; -import org.springframework.batch.core.configuration.annotation.BatchConfigurer; -import org.springframework.batch.core.configuration.annotation.DefaultBatchConfigurer; -import org.springframework.batch.core.explore.JobExplorer; -import org.springframework.batch.core.explore.support.MapJobExplorerFactoryBean; -import org.springframework.batch.core.repository.JobRepository; -import org.springframework.batch.core.repository.support.MapJobRepositoryFactoryBean; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import uk.ac.ebi.eva.commons.batch.job.JobExecutionApplicationListener; @@ -33,19 +27,4 @@ public class RunnerConfiguration { public JobExecutionApplicationListener jobExecutionApplicationListener() { return new JobExecutionApplicationListener(); } - - @Bean - public BatchConfigurer configurer() { - return new DefaultBatchConfigurer() { - @Override - protected JobRepository createJobRepository() throws Exception { - return new MapJobRepositoryFactoryBean().getObject(); - } - - @Override - protected JobExplorer createJobExplorer() throws Exception { - return new MapJobExplorerFactoryBean((MapJobRepositoryFactoryBean) new MapJobRepositoryFactoryBean()).getObject(); - } - }; - } } \ No newline at end of file diff --git a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/jobs/ActiveAccessionReleaseFromDBJobConfiguration.java b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/jobs/ActiveAccessionReleaseFromDBJobConfiguration.java index 7c8e522fb..fe595823c 100644 --- a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/jobs/ActiveAccessionReleaseFromDBJobConfiguration.java +++ b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/jobs/ActiveAccessionReleaseFromDBJobConfiguration.java @@ -3,8 +3,9 @@ import org.springframework.batch.core.Job; import org.springframework.batch.core.Step; import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; -import org.springframework.batch.core.configuration.annotation.JobBuilderFactory; +import org.springframework.batch.core.job.builder.JobBuilder; import org.springframework.batch.core.launch.support.RunIdIncrementer; +import org.springframework.batch.core.repository.JobRepository; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; @@ -22,8 +23,8 @@ public class ActiveAccessionReleaseFromDBJobConfiguration { private Step activeAccessionsReleaseFromDBStep; @Bean(ACTIVE_ACCESSIONS_RELEASE_FROM_DB_JOB) - public Job activeAccessionReleaseFromDBJob(JobBuilderFactory jobBuilderFactory) { - return jobBuilderFactory.get(ACTIVE_ACCESSIONS_RELEASE_FROM_DB_JOB) + public Job activeAccessionReleaseFromDBJob(JobRepository jobRepository) { + return new JobBuilder(ACTIVE_ACCESSIONS_RELEASE_FROM_DB_JOB, jobRepository) .incrementer(new RunIdIncrementer()) .start(activeAccessionsReleaseFromDBStep) .build(); diff --git a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/jobs/DumpRSAccessionsJobConfiguration.java b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/jobs/DumpRSAccessionsJobConfiguration.java index 9b7772816..e8be4c938 100644 --- a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/jobs/DumpRSAccessionsJobConfiguration.java +++ b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/jobs/DumpRSAccessionsJobConfiguration.java @@ -4,8 +4,9 @@ import org.springframework.batch.core.Job; import org.springframework.batch.core.Step; import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; -import org.springframework.batch.core.configuration.annotation.JobBuilderFactory; +import org.springframework.batch.core.job.builder.JobBuilder; import org.springframework.batch.core.launch.support.RunIdIncrementer; +import org.springframework.batch.core.repository.JobRepository; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; @@ -29,16 +30,16 @@ public class DumpRSAccessionsJobConfiguration { private Step dumpMergedAndDeprecatedAccessionsStep; @Bean(DUMP_ACTIVE_ACCESSIONS_JOB) - public Job dumpActiveAccessionJob(JobBuilderFactory jobBuilderFactory) { - return jobBuilderFactory.get(DUMP_ACTIVE_ACCESSIONS_JOB) + public Job dumpActiveAccessionJob(JobRepository jobRepository) { + return new JobBuilder(DUMP_ACTIVE_ACCESSIONS_JOB, jobRepository) .incrementer(new RunIdIncrementer()) .start(dumpActiveAccessionsStep) .build(); } @Bean(DUMP_MERGED_AND_DEPRECATED_ACCESSIONS_JOB) - public Job dumpMergedAndDeprecatedAccessionJob(JobBuilderFactory jobBuilderFactory) { - return jobBuilderFactory.get(DUMP_MERGED_AND_DEPRECATED_ACCESSIONS_JOB) + public Job dumpMergedAndDeprecatedAccessionJob(JobRepository jobRepository) { + return new JobBuilder(DUMP_MERGED_AND_DEPRECATED_ACCESSIONS_JOB, jobRepository) .incrementer(new RunIdIncrementer()) .start(dumpMergedAndDeprecatedAccessionsStep) .build(); diff --git a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/jobs/MergedAndDeprecatedAccessionReleaseFromDBJobConfiguration.java b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/jobs/MergedAndDeprecatedAccessionReleaseFromDBJobConfiguration.java index e442f067b..acf2c36b6 100644 --- a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/jobs/MergedAndDeprecatedAccessionReleaseFromDBJobConfiguration.java +++ b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/jobs/MergedAndDeprecatedAccessionReleaseFromDBJobConfiguration.java @@ -3,8 +3,9 @@ import org.springframework.batch.core.Job; import org.springframework.batch.core.Step; import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; -import org.springframework.batch.core.configuration.annotation.JobBuilderFactory; +import org.springframework.batch.core.job.builder.JobBuilder; import org.springframework.batch.core.launch.support.RunIdIncrementer; +import org.springframework.batch.core.repository.JobRepository; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; @@ -22,8 +23,8 @@ public class MergedAndDeprecatedAccessionReleaseFromDBJobConfiguration { private Step mergedAndDeprecatedAccessionsReleaseFromDBStep; @Bean(MERGED_AND_DEPRECATED_ACCESSIONS_RELEASE_FROM_DB_JOB) - public Job mergedAndDeprecatedAccessionReleaseFromDBJob(JobBuilderFactory jobBuilderFactory) { - return jobBuilderFactory.get(MERGED_AND_DEPRECATED_ACCESSIONS_RELEASE_FROM_DB_JOB) + public Job mergedAndDeprecatedAccessionReleaseFromDBJob(JobRepository jobRepository) { + return new JobBuilder(MERGED_AND_DEPRECATED_ACCESSIONS_RELEASE_FROM_DB_JOB, jobRepository) .incrementer(new RunIdIncrementer()) .start(mergedAndDeprecatedAccessionsReleaseFromDBStep) .build(); diff --git a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/processors/ReleaseProcessorConfiguration.java b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/processors/ReleaseProcessorConfiguration.java index ff0423234..acf161c86 100644 --- a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/processors/ReleaseProcessorConfiguration.java +++ b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/processors/ReleaseProcessorConfiguration.java @@ -21,14 +21,13 @@ import org.springframework.batch.item.support.CompositeItemProcessor; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; - -import uk.ac.ebi.eva.accession.core.contig.ContigMapping; import uk.ac.ebi.eva.accession.core.batch.io.FastaSynonymSequenceReader; -import uk.ac.ebi.eva.accession.release.parameters.InputParameters; +import uk.ac.ebi.eva.accession.core.contig.ContigMapping; import uk.ac.ebi.eva.accession.release.batch.processors.ContextNucleotideAdditionProcessor; import uk.ac.ebi.eva.accession.release.batch.processors.ExcludeInvalidVariantsProcessor; import uk.ac.ebi.eva.accession.release.batch.processors.NamedVariantProcessor; import uk.ac.ebi.eva.accession.release.batch.processors.VariantToVariantContextProcessor; +import uk.ac.ebi.eva.accession.release.parameters.InputParameters; import uk.ac.ebi.eva.commons.core.models.pipeline.Variant; import java.io.IOException; diff --git a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/steps/ActiveAccessionReleaseFromDBStepConfiguration.java b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/steps/ActiveAccessionReleaseFromDBStepConfiguration.java index ba9a87d33..be3a1e5d5 100644 --- a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/steps/ActiveAccessionReleaseFromDBStepConfiguration.java +++ b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/steps/ActiveAccessionReleaseFromDBStepConfiguration.java @@ -3,7 +3,8 @@ import htsjdk.variant.variantcontext.VariantContext; import org.springframework.batch.core.Step; import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; -import org.springframework.batch.core.configuration.annotation.StepBuilderFactory; +import org.springframework.batch.core.repository.JobRepository; +import org.springframework.batch.core.step.builder.StepBuilder; import org.springframework.batch.core.step.tasklet.TaskletStep; import org.springframework.batch.item.ItemProcessor; import org.springframework.batch.item.ItemReader; @@ -13,6 +14,7 @@ import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.transaction.PlatformTransactionManager; import uk.ac.ebi.eva.commons.core.models.pipeline.Variant; import static uk.ac.ebi.eva.accession.release.configuration.BeanNames.ACTIVE_ACCESSIONS_RELEASE_FROM_DB_STEP; @@ -37,10 +39,10 @@ public class ActiveAccessionReleaseFromDBStepConfiguration { ItemStreamWriter accessionWriter; @Bean(ACTIVE_ACCESSIONS_RELEASE_FROM_DB_STEP) - public Step activeAccessionsReleaseFromDBStep(StepBuilderFactory stepBuilderFactory, + public Step activeAccessionsReleaseFromDBStep(JobRepository jobRepository, PlatformTransactionManager transactionManager, SimpleCompletionPolicy chunkSizeCompletionPolicy) { - TaskletStep step = stepBuilderFactory.get(ACTIVE_ACCESSIONS_RELEASE_FROM_DB_STEP) - .chunk(chunkSizeCompletionPolicy) + TaskletStep step = new StepBuilder(ACTIVE_ACCESSIONS_RELEASE_FROM_DB_STEP, jobRepository) + .chunk(chunkSizeCompletionPolicy, transactionManager) .reader(variantReader) .processor(variantProcessor) .writer(accessionWriter) diff --git a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/steps/DumpRSAccessionsStepConfiguration.java b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/steps/DumpRSAccessionsStepConfiguration.java index abadaf279..762b22ac1 100644 --- a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/steps/DumpRSAccessionsStepConfiguration.java +++ b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/steps/DumpRSAccessionsStepConfiguration.java @@ -2,11 +2,13 @@ import org.springframework.batch.core.Step; import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; -import org.springframework.batch.core.configuration.annotation.StepBuilderFactory; +import org.springframework.batch.core.repository.JobRepository; +import org.springframework.batch.core.step.builder.StepBuilder; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.transaction.PlatformTransactionManager; import uk.ac.ebi.eva.accession.release.batch.io.DumpRSAccessionsInFile; import uk.ac.ebi.eva.accession.release.parameters.InputParameters; @@ -23,24 +25,26 @@ public class DumpRSAccessionsStepConfiguration { private DumpRSAccessionsInFile dumpRSAccessionsInFile; @Bean(DUMP_ACTIVE_ACCESSIONS_STEP) - public Step dumpActiveAccessionStep(StepBuilderFactory stepBuilderFactory, InputParameters inputParameters) { - return stepBuilderFactory.get(DUMP_ACTIVE_ACCESSIONS_STEP) + public Step dumpActiveAccessionStep(JobRepository jobRepository, PlatformTransactionManager transactionManager, + InputParameters inputParameters) { + return new StepBuilder(DUMP_ACTIVE_ACCESSIONS_STEP, jobRepository) .tasklet((contribution, chunkContext) -> { dumpRSAccessionsInFile.dumpAccessions(DumpRSAccessionsInFile.RSDumpType.ACTIVE, inputParameters.getAssemblyAccession()); return null; - }) + }, transactionManager) .build(); } @Bean(DUMP_MERGED_AND_DEPRECATED_ACCESSIONS_STEP) - public Step dumpMergedAndDeprecatedAccessionStep(StepBuilderFactory stepBuilderFactory, InputParameters inputParameters) { - return stepBuilderFactory.get(DUMP_MERGED_AND_DEPRECATED_ACCESSIONS_STEP) + public Step dumpMergedAndDeprecatedAccessionStep(JobRepository jobRepository, PlatformTransactionManager transactionManager, + InputParameters inputParameters) { + return new StepBuilder(DUMP_MERGED_AND_DEPRECATED_ACCESSIONS_STEP, jobRepository) .tasklet((contribution, chunkContext) -> { dumpRSAccessionsInFile.dumpAccessions(DumpRSAccessionsInFile.RSDumpType.MERGED_AND_DEPRECATED, inputParameters.getAssemblyAccession()); return null; - }) + }, transactionManager) .build(); } } \ No newline at end of file diff --git a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/steps/MergedAndDeprecatedAccessionReleaseFromDBStepConfiguration.java b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/steps/MergedAndDeprecatedAccessionReleaseFromDBStepConfiguration.java index 85f7893bc..a07221a2a 100644 --- a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/steps/MergedAndDeprecatedAccessionReleaseFromDBStepConfiguration.java +++ b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/steps/MergedAndDeprecatedAccessionReleaseFromDBStepConfiguration.java @@ -3,7 +3,8 @@ import htsjdk.variant.variantcontext.VariantContext; import org.springframework.batch.core.Step; import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; -import org.springframework.batch.core.configuration.annotation.StepBuilderFactory; +import org.springframework.batch.core.repository.JobRepository; +import org.springframework.batch.core.step.builder.StepBuilder; import org.springframework.batch.core.step.tasklet.TaskletStep; import org.springframework.batch.item.ItemProcessor; import org.springframework.batch.item.ItemReader; @@ -13,6 +14,7 @@ import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.transaction.PlatformTransactionManager; import uk.ac.ebi.eva.commons.core.models.pipeline.Variant; import static uk.ac.ebi.eva.accession.release.configuration.BeanNames.EVA_MERGED_RELEASE_WRITER; @@ -37,10 +39,10 @@ public class MergedAndDeprecatedAccessionReleaseFromDBStepConfiguration { ItemStreamWriter accessionWriter; @Bean(MERGED_AND_DEPRECATED_ACCESSIONS_RELEASE_FROM_DB_STEP) - public Step mergedAndDeprecatedAccessionsReleaseFromDBStep(StepBuilderFactory stepBuilderFactory, + public Step mergedAndDeprecatedAccessionsReleaseFromDBStep(JobRepository jobRepository, PlatformTransactionManager transactionManager, SimpleCompletionPolicy chunkSizeCompletionPolicy) { - TaskletStep step = stepBuilderFactory.get(MERGED_AND_DEPRECATED_ACCESSIONS_RELEASE_FROM_DB_STEP) - .chunk(chunkSizeCompletionPolicy) + TaskletStep step = new StepBuilder(MERGED_AND_DEPRECATED_ACCESSIONS_RELEASE_FROM_DB_STEP, jobRepository) + .chunk(chunkSizeCompletionPolicy, transactionManager) .reader(variantReader) .processor(variantProcessor) .writer(accessionWriter) diff --git a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/assembly/AssemblyNameRetrieverTest.java b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/assembly/AssemblyNameRetrieverTest.java index e201c80cb..a53499b9b 100644 --- a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/assembly/AssemblyNameRetrieverTest.java +++ b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/assembly/AssemblyNameRetrieverTest.java @@ -15,37 +15,32 @@ */ package uk.ac.ebi.eva.accession.release.assembly; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; - -import uk.ac.ebi.eva.accession.release.assembly.AssemblyNameRetriever; +import jakarta.xml.bind.JAXBContext; +import jakarta.xml.bind.JAXBException; +import jakarta.xml.bind.Unmarshaller; +import org.junit.jupiter.api.Test; +import uk.ac.ebi.eva.accession.core.utils.PipelineTemporaryFolderUtil; import uk.ac.ebi.eva.accession.release.assembly.AssemblyNameRetriever.EnaAssemblyXml; -import javax.xml.bind.JAXBContext; -import javax.xml.bind.JAXBException; -import javax.xml.bind.Unmarshaller; import java.io.File; import java.io.FileWriter; import java.io.IOException; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertThrows; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; public class AssemblyNameRetrieverTest { - - @Rule - public TemporaryFolder temporaryFolderRule = new TemporaryFolder(); + public PipelineTemporaryFolderUtil temporaryFolderUtil = new PipelineTemporaryFolderUtil(); @Test public void parseXml() throws IOException, JAXBException { - File xml = temporaryFolderRule.newFile(); + File xml = temporaryFolderUtil.newFile(); FileWriter fileWriter = new FileWriter(xml); fileWriter.write("\n" + - "GRCh38.p13\n" + - ""); + "GRCh38.p13\n" + + ""); fileWriter.close(); JAXBContext jaxbContext = JAXBContext.newInstance(EnaAssemblyXml.class); @@ -56,7 +51,7 @@ public void parseXml() throws IOException, JAXBException { @Test public void parseMissingName() throws IOException, JAXBException { - File xml = temporaryFolderRule.newFile(); + File xml = temporaryFolderUtil.newFile(); FileWriter fileWriter = new FileWriter(xml); fileWriter.write(""); fileWriter.close(); @@ -68,24 +63,24 @@ public void parseMissingName() throws IOException, JAXBException { } @Test - public void retrieve() throws IOException, JAXBException { + public void retrieve() { assertEquals("GRCh38.p13", new AssemblyNameRetriever("GCA_000001405.28").getAssemblyName().get()); } @Test - public void retrieveAssemblyWithWrongFormat() throws IOException, JAXBException { + public void retrieveAssemblyWithWrongFormat() { assertThrows(RuntimeException.class, () -> new AssemblyNameRetriever("GCA_wrong_format")); } @Test - public void retrieveNonExistentAssembly() throws IOException, JAXBException { + public void retrieveNonExistentAssembly() { assertFalse(new AssemblyNameRetriever("GCA_000000000.1").getAssemblyName().isPresent()); } @Test public void buildHumanReadableUrl() { assertEquals("https://www.ebi.ac.uk/ena/browser/view/GCA_000001405.28", - new AssemblyNameRetriever("GCA_000001405.28").buildAssemblyHumanReadableUrl()); + new AssemblyNameRetriever("GCA_000001405.28").buildAssemblyHumanReadableUrl()); } @Test diff --git a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/active/AccessionedVariantMongoReaderTest.java b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/active/AccessionedVariantMongoReaderTest.java index fce15442d..14bb1febe 100644 --- a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/active/AccessionedVariantMongoReaderTest.java +++ b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/active/AccessionedVariantMongoReaderTest.java @@ -15,30 +15,28 @@ */ package uk.ac.ebi.eva.accession.release.batch.io.active; -import com.lordofthejars.nosqlunit.annotation.UsingDataSet; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; -import com.mongodb.MongoClient; import com.mongodb.client.AggregateIterable; +import com.mongodb.client.MongoClient; import com.mongodb.client.MongoCollection; import com.mongodb.client.MongoCursor; import com.mongodb.client.MongoDatabase; import org.bson.Document; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.batch.item.ExecutionContext; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.ApplicationContext; +import org.springframework.core.io.ResourceLoader; +import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; - +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.MongoConfiguration; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; +import uk.ac.ebi.eva.accession.core.utils.MongoTestDataLoader; import uk.ac.ebi.eva.accession.release.collectionNames.DbsnpCollectionNames; import uk.ac.ebi.eva.accession.release.test.configuration.MongoTestConfiguration; -import uk.ac.ebi.eva.accession.release.test.rule.FixSpringMongoDbRule; import uk.ac.ebi.eva.commons.core.models.pipeline.Variant; import java.util.ArrayList; @@ -47,10 +45,10 @@ import java.util.Map; import java.util.stream.Collectors; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import static uk.ac.ebi.eva.accession.release.batch.io.VariantMongoAggregationReader.REMAPPED_KEY; import static uk.ac.ebi.eva.accession.release.batch.io.active.AccessionedVariantMongoReader.ALLELES_MATCH_KEY; import static uk.ac.ebi.eva.accession.release.batch.io.active.AccessionedVariantMongoReader.ASSEMBLY_MATCH_KEY; @@ -60,14 +58,10 @@ import static uk.ac.ebi.eva.accession.release.batch.io.active.AccessionedVariantMongoReader.SUPPORTED_BY_EVIDENCE_KEY; import static uk.ac.ebi.eva.accession.release.batch.io.active.AccessionedVariantMongoReader.VARIANT_CLASS_KEY; -@RunWith(SpringRunner.class) +@ExtendWith(SpringExtension.class) @TestPropertySource("classpath:application.properties") -@UsingDataSet(locations = { - "/test-data/dbsnpClusteredVariantEntity.json", - "/test-data/dbsnpSubmittedVariantEntity.json", - "/test-data/submittedVariantEntity.json"}) // includes 1 variant with dbsnp rsid @ContextConfiguration(classes = {MongoConfiguration.class, MongoTestConfiguration.class}) -public class AccessionedVariantMongoReaderTest { +public class AccessionedVariantMongoReaderTest extends MongoTestContainerHelper { private static final String ASSEMBLY_ACCESSION_1 = "GCA_000409795.2"; @@ -126,19 +120,28 @@ public class AccessionedVariantMongoReaderTest { @Autowired private MongoClient mongoClient; - //Required by nosql-unit @Autowired - private ApplicationContext applicationContext; + private MongoTemplate mongoTemplate; - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName(TEST_DB).build()); + @Autowired + private ResourceLoader resourceLoader; - @Before + @BeforeEach public void setUp() throws Exception { + mongoTemplate.getDb().drop(); executionContext = new ExecutionContext(); reader = new AccessionedVariantMongoReader(ASSEMBLY_ACCESSION_1, TAXONOMY_1, mongoClient, TEST_DB, CHUNK_SIZE, - new DbsnpCollectionNames()); + new DbsnpCollectionNames()); + + MongoTestDataLoader mongoTestDataLoader = new MongoTestDataLoader(mongoTemplate, resourceLoader); + mongoTestDataLoader.load("/test-data/dbsnpClusteredVariantEntity.json"); + mongoTestDataLoader.load("/test-data/dbsnpSubmittedVariantEntity.json"); + mongoTestDataLoader.load("/test-data/submittedVariantEntity.json"); + } + + @AfterEach + public void tearDown() { + mongoTemplate.getDb().drop(); } @Test @@ -146,9 +149,7 @@ public void readTestDataMongo() { MongoDatabase db = mongoClient.getDatabase(TEST_DB); MongoCollection collection = db.getCollection(DBSNP_CLUSTERED_VARIANT_ENTITY); - AggregateIterable result = collection.aggregate(reader.buildAggregation()) - .allowDiskUse(true) - .useCursor(true); + AggregateIterable result = collection.aggregate(reader.buildAggregation()).allowDiskUse(true); MongoCursor cursor = result.iterator(); @@ -158,15 +159,15 @@ public void readTestDataMongo() { variants.addAll(reader.getVariants(clusteredVariant)); } assertEquals(EXPECTED_LINES, variants.size()); - } + } @Test - public void reader() throws Exception { + public void reader() { List variants = readIntoList(); assertEquals(EXPECTED_LINES, variants.size()); } - private List readIntoList() throws Exception { + private List readIntoList() { reader.open(executionContext); List allVariants = new ArrayList<>(); List variants; @@ -178,7 +179,7 @@ private List readIntoList() throws Exception { } @Test - public void linkedSubmittedVariants() throws Exception { + public void linkedSubmittedVariants() { Map variants = readIntoMap(); assertEquals(EXPECTED_LINES, variants.size()); assertEquals(3, variants.values().stream().filter(v -> v.getMainId().equals(RS_1)).count()); @@ -188,7 +189,7 @@ public void linkedSubmittedVariants() throws Exception { assertEquals(1, variants.get(RS_2_T_G).getSourceEntries().size()); } - private Map readIntoMap() throws Exception { + private Map readIntoMap() { reader.open(executionContext); Map allVariants = new HashMap<>(); List variants; @@ -207,9 +208,9 @@ private String getStringId(Variant variant) { } @Test - public void queryOtherAssembly() throws Exception { + public void queryOtherAssembly() { reader = new AccessionedVariantMongoReader(ASSEMBLY_ACCESSION_2, TAXONOMY_2, mongoClient, TEST_DB, CHUNK_SIZE, - new DbsnpCollectionNames()); + new DbsnpCollectionNames()); Map variants = readIntoMap(); assertEquals(3, variants.size()); @@ -220,60 +221,60 @@ public void queryOtherAssembly() throws Exception { } @Test - public void snpVariantClassAttribute() throws Exception { + public void snpVariantClassAttribute() { Map variants = readIntoMap(); assertEquals(EXPECTED_LINES, variants.size()); String snpSequenceOntology = "SO:0001483"; assertTrue(variants - .get(RS_1_G_A) - .getSourceEntries() - .stream() - .allMatch(se -> snpSequenceOntology.equals(se.getAttribute(VARIANT_CLASS_KEY)))); + .get(RS_1_G_A) + .getSourceEntries() + .stream() + .allMatch(se -> snpSequenceOntology.equals(se.getAttribute(VARIANT_CLASS_KEY)))); assertTrue(variants - .get(RS_1_G_T) - .getSourceEntries() - .stream() - .allMatch(se -> snpSequenceOntology.equals(se.getAttribute(VARIANT_CLASS_KEY)))); + .get(RS_1_G_T) + .getSourceEntries() + .stream() + .allMatch(se -> snpSequenceOntology.equals(se.getAttribute(VARIANT_CLASS_KEY)))); assertTrue(variants - .get(RS_2_T_G) - .getSourceEntries() - .stream() - .allMatch(se -> snpSequenceOntology.equals(se.getAttribute(VARIANT_CLASS_KEY)))); + .get(RS_2_T_G) + .getSourceEntries() + .stream() + .allMatch(se -> snpSequenceOntology.equals(se.getAttribute(VARIANT_CLASS_KEY)))); } @Test - public void insertionVariantClassAttribute() throws Exception { + public void insertionVariantClassAttribute() { reader = new AccessionedVariantMongoReader(ASSEMBLY_ACCESSION_4, TAXONOMY_4, mongoClient, TEST_DB, CHUNK_SIZE, - new DbsnpCollectionNames()); + new DbsnpCollectionNames()); List variants = readIntoList(); assertEquals(1, variants.size()); String insertionSequenceOntology = "SO:0000667"; assertTrue(variants.get(0) - .getSourceEntries() - .stream() - .allMatch(se -> insertionSequenceOntology.equals(se.getAttribute(VARIANT_CLASS_KEY)))); + .getSourceEntries() + .stream() + .allMatch(se -> insertionSequenceOntology.equals(se.getAttribute(VARIANT_CLASS_KEY)))); } @Test - public void otherVariantClasses() throws Exception { + public void otherVariantClasses() { reader = new AccessionedVariantMongoReader(ASSEMBLY_ACCESSION_5, TAXONOMY_5, mongoClient, TEST_DB, CHUNK_SIZE, - new DbsnpCollectionNames()); + new DbsnpCollectionNames()); List variants = readIntoList(); assertEquals(4, variants.size()); String indelSequenceOntology = "SO:1000032"; String tandemRepeatSequenceOntology = "SO:0000705"; assertEquals(3, variants.stream() - .flatMap(v -> v.getSourceEntries().stream()) - .filter(se -> tandemRepeatSequenceOntology.equals(se.getAttribute(VARIANT_CLASS_KEY))) - .count()); + .flatMap(v -> v.getSourceEntries().stream()) + .filter(se -> tandemRepeatSequenceOntology.equals(se.getAttribute(VARIANT_CLASS_KEY))) + .count()); assertEquals(1, variants.stream() - .flatMap(v -> v.getSourceEntries().stream()) - .filter(se -> indelSequenceOntology.equals(se.getAttribute(VARIANT_CLASS_KEY))) - .count()); + .flatMap(v -> v.getSourceEntries().stream()) + .filter(se -> indelSequenceOntology.equals(se.getAttribute(VARIANT_CLASS_KEY))) + .count()); } @Test - public void studyIdAttribute() throws Exception { + public void studyIdAttribute() { Map variants = readIntoMap(); assertEquals(EXPECTED_LINES, variants.size()); @@ -287,9 +288,9 @@ public void studyIdAttribute() throws Exception { } @Test - public void clusteredVariantWithoutSubmittedVariants() throws Exception { + public void clusteredVariantWithoutSubmittedVariants() { reader = new AccessionedVariantMongoReader(ASSEMBLY_ACCESSION_3, TAXONOMY_3, mongoClient, TEST_DB, CHUNK_SIZE, - new DbsnpCollectionNames()); + new DbsnpCollectionNames()); List variants = readIntoList(); assertEquals(0, variants.size()); } @@ -301,105 +302,105 @@ public void includeValidatedFlag() throws Exception { } @Test - public void includeRemappedFlag() throws Exception { + public void includeRemappedFlag() { List variants = readIntoList(); assertNotEquals(0, variants.size()); List rsToLookFor = variants.stream().filter(v -> v.getMainId().equals("rs8181")) - .collect(Collectors.toList()); + .collect(Collectors.toList()); assertNotEquals(0, rsToLookFor.size()); // Both the SS clustered by the RS 8181 has the "remappedFrom" attribute assertTrue(rsToLookFor.stream().flatMap(v -> v.getSourceEntries().stream()) - .map(se -> se.getAttribute(REMAPPED_KEY)) - .map(Boolean::new) - .allMatch(v -> v.equals(true))); + .map(se -> se.getAttribute(REMAPPED_KEY)) + .map(Boolean::parseBoolean) + .allMatch(v -> v.equals(true))); rsToLookFor = variants.stream().filter(v -> v.getMainId().equals("rs869808637")) - .collect(Collectors.toList()); + .collect(Collectors.toList()); assertNotEquals(0, rsToLookFor.size()); // Only one of the SS clustered by the RS 869808637 has the "remappedFrom" attribute assertTrue(rsToLookFor.stream().flatMap(v -> v.getSourceEntries().stream()) - .map(se -> se.getAttribute(REMAPPED_KEY)) - .map(Boolean::new) - .allMatch(v -> v.equals(false))); + .map(se -> se.getAttribute(REMAPPED_KEY)) + .map(Boolean::parseBoolean) + .allMatch(v -> v.equals(false))); } - private void assertFlagEqualsInAllVariants(String key, boolean value) throws Exception { + private void assertFlagEqualsInAllVariants(String key, boolean value) { List variants = readIntoList(); assertNotEquals(0, variants.size()); assertTrue(variants.stream() - .flatMap(v -> v.getSourceEntries().stream()) - .map(se -> se.getAttribute(key)) - .map(Boolean::new) - .allMatch(v -> v.equals(value))); + .flatMap(v -> v.getSourceEntries().stream()) + .map(se -> se.getAttribute(key)) + .map(Boolean::parseBoolean) + .allMatch(v -> v.equals(value))); } @Test - public void includeAssemblyMatchFlag() throws Exception { + public void includeAssemblyMatchFlag() { assertFlagEqualsInAllVariants(ASSEMBLY_MATCH_KEY, true); } @Test - public void includeAllelesMatchFlag() throws Exception { + public void includeAllelesMatchFlag() { assertFlagEqualsInAllVariants(ALLELES_MATCH_KEY, true); } @Test - public void includeEvidenceFlag() throws Exception { + public void includeEvidenceFlag() { assertFlagEqualsInAllVariants(SUPPORTED_BY_EVIDENCE_KEY, true); } @Test - public void includeValidatedNonDefaultFlag() throws Exception { + public void includeValidatedNonDefaultFlag() { reader = new AccessionedVariantMongoReader(ASSEMBLY_ACCESSION_5, TAXONOMY_5, mongoClient, TEST_DB, CHUNK_SIZE, - new DbsnpCollectionNames()); + new DbsnpCollectionNames()); assertFlagEqualsInAllVariants(SUBMITTED_VARIANT_VALIDATED_KEY, true); assertFlagEqualsInRS(CLUSTERED_VARIANT_VALIDATED_KEY, false, RS_4); assertFlagEqualsInRS(CLUSTERED_VARIANT_VALIDATED_KEY, true, RS_5); } - private void assertFlagEqualsInRS(String key, boolean value, String clusteredVariantAccession) throws Exception { + private void assertFlagEqualsInRS(String key, boolean value, String clusteredVariantAccession) { List variants = readIntoList(); assertNotEquals(0, variants.size()); assertTrue(variants.stream() - .filter(v -> v.getMainId().equals(clusteredVariantAccession)) - .flatMap(v -> v.getSourceEntries().stream()) - .map(se -> se.getAttribute(key)) - .map(Boolean::new) - .allMatch(v -> v.equals(value))); + .filter(v -> v.getMainId().equals(clusteredVariantAccession)) + .flatMap(v -> v.getSourceEntries().stream()) + .map(se -> se.getAttribute(key)) + .map(Boolean::parseBoolean) + .allMatch(v -> v.equals(value))); } @Test - public void includeAssemblyMatchNonDefaultFlag() throws Exception { + public void includeAssemblyMatchNonDefaultFlag() { reader = new AccessionedVariantMongoReader(ASSEMBLY_ACCESSION_4, TAXONOMY_4, mongoClient, TEST_DB, CHUNK_SIZE, - new DbsnpCollectionNames()); + new DbsnpCollectionNames()); assertFlagEqualsInAllVariants(ASSEMBLY_MATCH_KEY, false); } @Test - public void includeAllelesMatchNonDefaultFlag() throws Exception { + public void includeAllelesMatchNonDefaultFlag() { reader = new AccessionedVariantMongoReader(ASSEMBLY_ACCESSION_4, TAXONOMY_4, mongoClient, TEST_DB, CHUNK_SIZE, - new DbsnpCollectionNames()); + new DbsnpCollectionNames()); assertFlagEqualsInAllVariants(ALLELES_MATCH_KEY, false); } @Test - public void includeEvidenceNonDefaultFlag() throws Exception { + public void includeEvidenceNonDefaultFlag() { reader = new AccessionedVariantMongoReader(ASSEMBLY_ACCESSION_5, TAXONOMY_5, mongoClient, TEST_DB, CHUNK_SIZE, - new DbsnpCollectionNames()); + new DbsnpCollectionNames()); assertFlagEqualsInAllVariants(SUPPORTED_BY_EVIDENCE_KEY, false); } /** * Two clustered variants with the same accession but mapped against different locations. Each clustered variant * should only appear with the alleles of the its corresponding submitted variants. - * + *

* This means variants will only be returned by the reader when the clustered and submitted variant have the same location * (contig and start) */ @Test - public void includeOnlyVariantsWithTheSameChromosomeAndStartInRsAndSs() throws Exception { + public void includeOnlyVariantsWithTheSameChromosomeAndStartInRsAndSs() { reader = new AccessionedVariantMongoReader("GCA_000002775.1", 3694, mongoClient, - TEST_DB, CHUNK_SIZE, new DbsnpCollectionNames()); + TEST_DB, CHUNK_SIZE, new DbsnpCollectionNames()); List allVariants = readIntoList(); assertEquals(3, allVariants.size()); @@ -423,31 +424,31 @@ private boolean isVariantPresent(List variants, String chromosome, long * For ambiguous INDELS the start position in the clustered variant and its submitted variants can be different * because the renormalization process is performed only for submitted variants. this will be handled by trying to * match with the exact position or either the one before or after. - * + *

* Variants are represented in different ways by dbSNP and the EVA * dbSNP (start: 7356605, reference: , alternate: GAGCTATGATCTTCGGAAGGAGAAGGAGAAGGAAAAGATTCATGACGTCCACA) * EVA (start: 7356604, reference: , alternate: AGAGCTATGATCTTCGGAAGGAGAAGGAGAAGGAAAAGATTCATGACGTCCAC) - * + *

* FASTA (NC_024803.1:7356603-7356606) TATC - * + *

* dbSNP remove the context nucleotide before an INDEL while the EVA removes the rightmost bases - * + *

* dbSNP: TA(GAGCTATGATCTTCGGAAGGAGAAGGAGAAGGAAAAGATTCATGACGTCCACA)TC, start: 7356605 * EVA: T(AGAGCTATGATCTTCGGAAGGAGAAGGAGAAGGAAAAGATTCATGACGTCCAC)ATC, start: 7356604 - * + *

* For the rs268233057 (start: 7356605) and its ss490570267 (start: 7356604) the submitted variant start will be * used along with its alleles even when the start position does not exactly match. */ @Test - public void includeAmbiguousVariantsWithDifferentStartInSsAndRs() throws Exception { + public void includeAmbiguousVariantsWithDifferentStartInSsAndRs() { reader = new AccessionedVariantMongoReader(ASSEMBLY_ACCESSION_4, TAXONOMY_4, mongoClient, TEST_DB, CHUNK_SIZE, - new DbsnpCollectionNames()); + new DbsnpCollectionNames()); List allVariants = readIntoList(); assertEquals(1, allVariants.size()); assertTrue(isVariantPresent(allVariants, "CM001642.1", 7356604L, "", - "AGAGCTATGATCTTCGGAAGGAGAAGGAGAAGGAAAAGATTCATGACGTCCAC")); + "AGAGCTATGATCTTCGGAAGGAGAAGGAGAAGGAAAAGATTCATGACGTCCAC")); } /* @@ -455,9 +456,9 @@ public void includeAmbiguousVariantsWithDifferentStartInSsAndRs() throws Excepti ensure that the variant list only uses the ss1 entry in ASM1 */ @Test - public void ensureOnlySSInReleaseAssemblyIsUsed() throws Exception { + public void ensureOnlySSInReleaseAssemblyIsUsed() { Variant variantInTwoAssemblies = readIntoList().stream().filter(e -> e.getIds().contains("rs100")) - .findFirst().get(); + .findFirst().get(); // Ensure that only one SS entry is available in the variant that was read assertEquals(1, variantInTwoAssemblies.getSourceEntries().size()); } diff --git a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/active/VariantContextWriterTest.java b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/active/VariantContextWriterTest.java index ac7c66585..49b863d2f 100644 --- a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/active/VariantContextWriterTest.java +++ b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/active/VariantContextWriterTest.java @@ -16,16 +16,15 @@ package uk.ac.ebi.eva.accession.release.batch.io.active; import htsjdk.variant.variantcontext.VariantContext; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; - +import org.junit.jupiter.api.Test; +import org.springframework.batch.item.Chunk; import uk.ac.ebi.eva.accession.core.contig.ContigMapping; import uk.ac.ebi.eva.accession.core.contig.ContigNaming; import uk.ac.ebi.eva.accession.core.contig.ContigSynonyms; +import uk.ac.ebi.eva.accession.core.utils.PipelineTemporaryFolderUtil; import uk.ac.ebi.eva.accession.release.batch.io.contig.ContigWriter; -import uk.ac.ebi.eva.accession.release.parameters.ReportPathResolver; import uk.ac.ebi.eva.accession.release.batch.processors.VariantToVariantContextProcessor; +import uk.ac.ebi.eva.accession.release.parameters.ReportPathResolver; import uk.ac.ebi.eva.commons.core.models.pipeline.Variant; import uk.ac.ebi.eva.commons.core.models.pipeline.VariantSourceEntry; @@ -45,8 +44,9 @@ import java.util.stream.Collectors; import java.util.stream.Stream; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import static uk.ac.ebi.eva.accession.release.batch.io.active.AccessionedVariantMongoReader.ALLELES_MATCH_KEY; import static uk.ac.ebi.eva.accession.release.batch.io.active.AccessionedVariantMongoReader.ASSEMBLY_MATCH_KEY; import static uk.ac.ebi.eva.accession.release.batch.io.active.AccessionedVariantMongoReader.CLUSTERED_VARIANT_VALIDATED_KEY; @@ -98,22 +98,21 @@ public class VariantContextWriterTest { // inner string without space nor angle brackets, surrounded by angle brackets private static final String SYMBOLIC_ALLELE_REGEX = "<[^<> ]+>"; - @Rule - public TemporaryFolder temporaryFolder = new TemporaryFolder(); + public PipelineTemporaryFolderUtil temporaryFolderUtil = new PipelineTemporaryFolderUtil(); private static final String DATA_LINES_REGEX = "^[^#].*"; @Test public void basicWrite() throws Exception { - File outputFolder = temporaryFolder.newFolder(); + File outputFolder = temporaryFolderUtil.newFolder(); File output = assertWriteVcf(outputFolder, - buildVariant(GENBANK_ACCESSION_1, 1000, "C", "A", SNP_SEQUENCE_ONTOLOGY, STUDY_1)); + buildVariant(GENBANK_ACCESSION_1, 1000, "C", "A", SNP_SEQUENCE_ONTOLOGY, STUDY_1)); } - private Variant buildVariant(String chr, int start, String reference, String alternate,String sequenceOntologyTerm, + private Variant buildVariant(String chr, int start, String reference, String alternate, String sequenceOntologyTerm, String... studies) { return buildVariant(chr, start, reference, alternate, sequenceOntologyTerm, false, false, true, true, true, - studies); + studies); } private Variant buildVariant(String chr, int start, String reference, String alternate, String sequenceOntologyTerm, @@ -138,7 +137,7 @@ private Variant buildVariant(String chr, int start, String reference, String alt private File assertWriteVcf(File outputFolder, Variant... variants) throws Exception { Path reportPath = ReportPathResolver.getDbsnpCurrentIdsReportPath(outputFolder.getAbsolutePath(), REFERENCE_ASSEMBLY); String activeContigsFilePath = ContigWriter.getDbsnpActiveContigsFilePath(reportPath.toFile().getParent(), - REFERENCE_ASSEMBLY); + REFERENCE_ASSEMBLY); VariantContextWriter writer = new VariantContextWriter(reportPath, REFERENCE_ASSEMBLY, activeContigsFilePath); writer.open(null); @@ -150,7 +149,7 @@ private File assertWriteVcf(File outputFolder, Variant... variants) throws Excep List variantContexts = Stream.of(variants).map(variantToVariantContextProcessor::process).collect(Collectors.toList()); - writer.write(variantContexts); + writer.write(new Chunk<>(variantContexts)); writer.close(); @@ -162,14 +161,14 @@ private File assertWriteVcf(File outputFolder, Variant... variants) throws Excep @Test public void checkReference() throws Exception { - File outputFolder = temporaryFolder.newFolder(); + File outputFolder = temporaryFolderUtil.newFolder(); File output = assertWriteVcf(outputFolder, - buildVariant(GENBANK_ACCESSION_1, 1000, "C", "A", SNP_SEQUENCE_ONTOLOGY, STUDY_1)); + buildVariant(GENBANK_ACCESSION_1, 1000, "C", "A", SNP_SEQUENCE_ONTOLOGY, STUDY_1)); List referenceLines = grepFile(output, "^##reference.*"); assertEquals(1, referenceLines.size()); assertEquals("##reference=", referenceLines.get(0)); + + REFERENCE_ASSEMBLY + ">", referenceLines.get(0)); } private List grepFile(File file, String regex) throws IOException { @@ -187,7 +186,7 @@ private List grepFile(File file, String regex) throws IOException { @Test public void checkMetadataSection() throws Exception { - File outputFolder = temporaryFolder.newFolder(); + File outputFolder = temporaryFolderUtil.newFolder(); FileWriter fileWriter = new FileWriter( ContigWriter.getDbsnpActiveContigsFilePath(outputFolder.getAbsolutePath(), REFERENCE_ASSEMBLY)); String contig = "CM0001.1,Chr1"; @@ -195,7 +194,7 @@ public void checkMetadataSection() throws Exception { fileWriter.close(); File output = assertWriteVcf(outputFolder, - buildVariant(GENBANK_ACCESSION_1, 1000, "C", "A", SNP_SEQUENCE_ONTOLOGY, STUDY_1)); + buildVariant(GENBANK_ACCESSION_1, 1000, "C", "A", SNP_SEQUENCE_ONTOLOGY, STUDY_1)); List metadataLines = grepFile(output, "^##.*"); assertEquals(11, metadataLines.size()); @@ -207,9 +206,9 @@ public void checkMetadataSection() throws Exception { @Test public void checkAccession() throws Exception { - File outputFolder = temporaryFolder.newFolder(); + File outputFolder = temporaryFolderUtil.newFolder(); File output = assertWriteVcf(outputFolder, - buildVariant(GENBANK_ACCESSION_1, 1000, "C", "A", SNP_SEQUENCE_ONTOLOGY, STUDY_1)); + buildVariant(GENBANK_ACCESSION_1, 1000, "C", "A", SNP_SEQUENCE_ONTOLOGY, STUDY_1)); List dataLines = grepFileContains(output, ID); assertEquals(1, dataLines.size()); @@ -217,9 +216,9 @@ public void checkAccession() throws Exception { @Test public void checkColumns() throws Exception { - File outputFolder = temporaryFolder.newFolder(); + File outputFolder = temporaryFolderUtil.newFolder(); File output = assertWriteVcf(outputFolder, - buildVariant(GENBANK_ACCESSION_1, 1000, "C", "A", SNP_SEQUENCE_ONTOLOGY, STUDY_1)); + buildVariant(GENBANK_ACCESSION_1, 1000, "C", "A", SNP_SEQUENCE_ONTOLOGY, STUDY_1)); List dataLines = grepFileContains(output, ID); assertEquals(1, dataLines.size()); @@ -228,10 +227,10 @@ public void checkColumns() throws Exception { @Test public void checkStudies() throws Exception { - File outputFolder = temporaryFolder.newFolder(); + File outputFolder = temporaryFolderUtil.newFolder(); File output = assertWriteVcf(outputFolder, - buildVariant(GENBANK_ACCESSION_1, 1000, "C", "A", SNP_SEQUENCE_ONTOLOGY, STUDY_1, - STUDY_2)); + buildVariant(GENBANK_ACCESSION_1, 1000, "C", "A", SNP_SEQUENCE_ONTOLOGY, STUDY_1, + STUDY_2)); List metadataLines = grepFileContains(output, STUDY_1); assertEquals(1, metadataLines.size()); @@ -252,10 +251,10 @@ public void checkSnpSequenceOntology() throws Exception { } private void checkSequenceOntology(String sequenceOntology, String reference, String alternate) throws Exception { - File outputFolder = temporaryFolder.newFolder(); + File outputFolder = temporaryFolderUtil.newFolder(); File output = assertWriteVcf(outputFolder, - buildVariant(GENBANK_ACCESSION_1, 1000, reference, alternate, sequenceOntology, - STUDY_1, STUDY_2)); + buildVariant(GENBANK_ACCESSION_1, 1000, reference, alternate, sequenceOntology, + STUDY_1, STUDY_2)); String dataWithVariantClassRegex = createRegexSurroundedByTabOrSemicolonInDataLine( VARIANT_CLASS_KEY + "=SO:[0-9]+"); @@ -292,9 +291,9 @@ public void checkTandemRepeatSequenceOntology() throws Exception { checkSequenceOntology(TANDEM_REPEAT_SEQUENCE_ONTOLOGY, "C", "CAGAG"); } - @Test(expected = IllegalArgumentException.class) - public void checkSequenceAlterationSequenceOntology() throws Exception { - checkSequenceOntology(SEQUENCE_ALTERATION_SEQUENCE_ONTOLOGY, "(ADL260)", "(LEI0062)"); + @Test + public void checkSequenceAlterationSequenceOntology() { + assertThrows(IllegalArgumentException.class, () -> checkSequenceOntology(SEQUENCE_ALTERATION_SEQUENCE_ONTOLOGY, "(ADL260)", "(LEI0062)")); } @Test @@ -304,7 +303,7 @@ public void checkMnvSequenceOntology() throws Exception { @Test public void checkSeveralVariants() throws Exception { - File outputFolder = temporaryFolder.newFolder(); + File outputFolder = temporaryFolderUtil.newFolder(); int position1 = 1003; int position2 = 1003; int position3 = 1002; @@ -313,12 +312,12 @@ public void checkSeveralVariants() throws Exception { String alternate3 = "G"; File output = assertWriteVcf(outputFolder, - buildVariant(GENBANK_ACCESSION_1, position1, "C", alternate1, - SNP_SEQUENCE_ONTOLOGY, STUDY_1), - buildVariant(GENBANK_ACCESSION_1, position2, "C", alternate2, - SNP_SEQUENCE_ONTOLOGY, STUDY_1), - buildVariant(GENBANK_ACCESSION_1, position3, "C", alternate3, - SNP_SEQUENCE_ONTOLOGY, STUDY_1)); + buildVariant(GENBANK_ACCESSION_1, position1, "C", alternate1, + SNP_SEQUENCE_ONTOLOGY, STUDY_1), + buildVariant(GENBANK_ACCESSION_1, position2, "C", alternate2, + SNP_SEQUENCE_ONTOLOGY, STUDY_1), + buildVariant(GENBANK_ACCESSION_1, position3, "C", alternate3, + SNP_SEQUENCE_ONTOLOGY, STUDY_1)); List dataLines = grepFileContains(output, ID); assertEquals(3, dataLines.size()); @@ -334,29 +333,29 @@ public void checkSeveralVariants() throws Exception { @Test public void checkStandardNucleotides() throws Exception { - File outputFolder = temporaryFolder.newFolder(); + File outputFolder = temporaryFolderUtil.newFolder(); File output = assertWriteVcf(outputFolder, - buildVariant(GENBANK_ACCESSION_1, 1000, "C", "NACTG", SNP_SEQUENCE_ONTOLOGY, - STUDY_1)); + buildVariant(GENBANK_ACCESSION_1, 1000, "C", "NACTG", SNP_SEQUENCE_ONTOLOGY, + STUDY_1)); List dataLines = grepFileContains(output, ID); assertEquals(1, dataLines.size()); } - @Test(expected = IllegalArgumentException.class) + @Test public void throwIfNonStandardNucleotides() throws Exception { - File outputFolder = temporaryFolder.newFolder(); - File output = assertWriteVcf(outputFolder, - buildVariant(GENBANK_ACCESSION_1, 1000, "C", "U", SNP_SEQUENCE_ONTOLOGY, STUDY_1)); + File outputFolder = temporaryFolderUtil.newFolder(); + assertThrows(IllegalArgumentException.class, () -> assertWriteVcf(outputFolder, + buildVariant(GENBANK_ACCESSION_1, 1000, "C", "U", SNP_SEQUENCE_ONTOLOGY, STUDY_1))); } @Test public void checkFlagsAreNotPresentWhenDefaultValues() throws Exception { - File outputFolder = temporaryFolder.newFolder(); + File outputFolder = temporaryFolderUtil.newFolder(); File output = assertWriteVcf(outputFolder, - buildVariant(GENBANK_ACCESSION_1, 1000, "C", "G", SNP_SEQUENCE_ONTOLOGY, false, - false, true, true, true, - STUDY_1)); + buildVariant(GENBANK_ACCESSION_1, 1000, "C", "G", SNP_SEQUENCE_ONTOLOGY, false, + false, true, true, true, + STUDY_1)); String dataLinesWithFlagsRegex = DATA_LINES_REGEX + "(" + CLUSTERED_VARIANT_VALIDATED_KEY + "|" + ALLELES_MATCH_KEY @@ -373,11 +372,11 @@ public void checkNonDefaultValidatedFlag() throws Exception { } private void assertFlagIsPresent(String flagRegex) throws Exception { - File outputFolder = temporaryFolder.newFolder(); + File outputFolder = temporaryFolderUtil.newFolder(); File output = assertWriteVcf(outputFolder, - buildVariant(GENBANK_ACCESSION_1, 1000, "C", "G", SNP_SEQUENCE_ONTOLOGY, true, - true, false, false, false, - STUDY_1)); + buildVariant(GENBANK_ACCESSION_1, 1000, "C", "G", SNP_SEQUENCE_ONTOLOGY, true, + true, false, false, false, + STUDY_1)); String dataLinesWithValidatedRegex = createRegexSurroundedByTabOrSemicolonInDataLine(flagRegex); List dataLines; @@ -429,7 +428,7 @@ private void assertSeveralFlagValues(String flagKey, boolean firstValue, boolean VariantSourceEntry sourceEntry2 = new VariantSourceEntry(STUDY_2, FILE_ID); sourceEntry2.addAttribute(flagKey, Boolean.toString(secondValue)); variant.addSourceEntry(sourceEntry2); - File outputFolder = temporaryFolder.newFolder(); + File outputFolder = temporaryFolderUtil.newFolder(); File output = assertWriteVcf(outputFolder, variant); @@ -460,9 +459,9 @@ public void writeNamedInsertion() throws Exception { } private void assertNamedVariant(String reference, String alternate) throws Exception { - File outputFolder = temporaryFolder.newFolder(); + File outputFolder = temporaryFolderUtil.newFolder(); File output = assertWriteVcf(outputFolder, buildVariant(GENBANK_ACCESSION_1, 1000, reference, alternate, - SEQUENCE_ALTERATION_SEQUENCE_ONTOLOGY, STUDY_1)); + SEQUENCE_ALTERATION_SEQUENCE_ONTOLOGY, STUDY_1)); List dataLines = grepFile(output, DATA_LINES_REGEX); assertEquals(1, dataLines.size()); @@ -476,35 +475,35 @@ public void writeNamedDeletion() throws Exception { assertNamedVariant("A", "<1190_BP_DEL>"); } - @Test(expected = IllegalArgumentException.class) + @Test public void throwIfNamedReference() throws Exception { - File outputFolder = temporaryFolder.newFolder(); - File output = assertWriteVcf(outputFolder, buildVariant(GENBANK_ACCESSION_1, 1000, "<1190_BP_DEL>", "A", - SEQUENCE_ALTERATION_SEQUENCE_ONTOLOGY, STUDY_1)); + File outputFolder = temporaryFolderUtil.newFolder(); + assertThrows(IllegalArgumentException.class, () -> assertWriteVcf(outputFolder, buildVariant(GENBANK_ACCESSION_1, 1000, "<1190_BP_DEL>", "A", + SEQUENCE_ALTERATION_SEQUENCE_ONTOLOGY, STUDY_1))); } @Test public void writeStudyWithInvalidCharacters() throws Exception { - File outputFolder = temporaryFolder.newFolder(); + File outputFolder = temporaryFolderUtil.newFolder(); File output = assertWriteVcf(outputFolder, - buildVariant(GENBANK_ACCESSION_1, 1000, "A", "T", - SEQUENCE_ALTERATION_SEQUENCE_ONTOLOGY, - "study_id=weird, yes; but not impossible in dbSNP"), - buildVariant(GENBANK_ACCESSION_1, 2000, "A", "T", - SEQUENCE_ALTERATION_SEQUENCE_ONTOLOGY, - "one =study= ", - ";; extra study ;;")); + buildVariant(GENBANK_ACCESSION_1, 1000, "A", "T", + SEQUENCE_ALTERATION_SEQUENCE_ONTOLOGY, + "study_id=weird, yes; but not impossible in dbSNP"), + buildVariant(GENBANK_ACCESSION_1, 2000, "A", "T", + SEQUENCE_ALTERATION_SEQUENCE_ONTOLOGY, + "one =study= ", + ";; extra study ;;")); List dataLines = grepFile(output, DATA_LINES_REGEX); assertEquals(2, dataLines.size()); String[] columns = dataLines.get(0).split("\t"); assertEquals(Arrays.asList("study_id_weird__yes__but_not_impossible_in_dbSNP"), - parseInfoFields(columns[INFO_COLUMN]).get(STUDY_ID_KEY)); + parseInfoFields(columns[INFO_COLUMN]).get(STUDY_ID_KEY)); columns = dataLines.get(1).split("\t"); assertEquals(new HashSet<>(Arrays.asList("one__study__", "___extra_study___")), - new HashSet<>(parseInfoFields(columns[INFO_COLUMN]).get(STUDY_ID_KEY))); + new HashSet<>(parseInfoFields(columns[INFO_COLUMN]).get(STUDY_ID_KEY))); } private HashMap> parseInfoFields(String column) { diff --git a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/contig/ContigMongoReaderTest.java b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/contig/ContigMongoReaderTest.java index 4df8bf83d..17056b906 100644 --- a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/contig/ContigMongoReaderTest.java +++ b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/contig/ContigMongoReaderTest.java @@ -16,42 +16,36 @@ package uk.ac.ebi.eva.accession.release.batch.io.contig; -import com.lordofthejars.nosqlunit.annotation.UsingDataSet; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; -import com.mongodb.MongoClient; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; +import com.mongodb.client.MongoClient; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.batch.item.ExecutionContext; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.ApplicationContext; +import org.springframework.core.io.ResourceLoader; +import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; - +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.MongoConfiguration; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; +import uk.ac.ebi.eva.accession.core.utils.MongoTestDataLoader; import uk.ac.ebi.eva.accession.release.collectionNames.DbsnpCollectionNames; import uk.ac.ebi.eva.accession.release.collectionNames.EvaCollectionNames; import uk.ac.ebi.eva.accession.release.test.configuration.MongoTestConfiguration; -import uk.ac.ebi.eva.accession.release.test.rule.FixSpringMongoDbRule; import java.util.ArrayList; import java.util.Arrays; import java.util.HashSet; import java.util.List; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; -@RunWith(SpringRunner.class) +@ExtendWith(SpringExtension.class) @TestPropertySource("classpath:application.properties") -@UsingDataSet(locations = { - "/test-data/dbsnpClusteredVariantEntity.json", - "/test-data/dbsnpClusteredVariantOperationEntity.json", - "/test-data/submittedVariantEntity.json" -}) @ContextConfiguration(classes = {MongoConfiguration.class, MongoTestConfiguration.class}) -public class ContigMongoReaderTest { +public class ContigMongoReaderTest extends MongoTestContainerHelper { private static final String TEST_DB = "test-db"; @@ -62,13 +56,25 @@ public class ContigMongoReaderTest { @Autowired private MongoClient mongoClient; - //Required by nosql-unit @Autowired - private ApplicationContext applicationContext; + private MongoTemplate mongoTemplate; + + @Autowired + private ResourceLoader resourceLoader; - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName(TEST_DB).build()); + @BeforeEach + public void setUp() { + mongoTemplate.getDb().drop(); + MongoTestDataLoader mongoTestDataLoader = new MongoTestDataLoader(mongoTemplate, resourceLoader); + mongoTestDataLoader.load("/test-data/dbsnpClusteredVariantEntity.json"); + mongoTestDataLoader.load("/test-data/dbsnpClusteredVariantOperationEntity.json"); + mongoTestDataLoader.load("/test-data/submittedVariantEntity.json"); + } + + @AfterEach + public void tearDown() { + mongoTemplate.getDb().drop(); + } @Test public void basicActiveContigsRead() { @@ -86,7 +92,7 @@ public void basicActiveContigsRead() { @Test public void basicMergedContigsRead() { ContigMongoReader reader = ContigMongoReader.mergedContigReader(ASSEMBLY_ACCESSION, mongoClient, - TEST_DB, new DbsnpCollectionNames()); + TEST_DB, new DbsnpCollectionNames()); reader.open(new ExecutionContext()); String contig; List contigs = new ArrayList<>(); @@ -99,7 +105,7 @@ public void basicMergedContigsRead() { @Test public void basicMultimapContigsRead() { ContigMongoReader reader = ContigMongoReader.multimapContigReader(ASSEMBLY_ACCESSION, mongoClient, TEST_DB, - new DbsnpCollectionNames()); + new DbsnpCollectionNames()); reader.open(new ExecutionContext()); String contig; List contigs = new ArrayList<>(); diff --git a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/contig/ContigWriterTest.java b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/contig/ContigWriterTest.java index 689c520e3..31231b82b 100644 --- a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/contig/ContigWriterTest.java +++ b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/contig/ContigWriterTest.java @@ -15,14 +15,12 @@ */ package uk.ac.ebi.eva.accession.release.batch.io.contig; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; - +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.batch.item.Chunk; import uk.ac.ebi.eva.accession.core.contig.ContigMapping; import uk.ac.ebi.eva.accession.core.contig.ContigSynonyms; -import uk.ac.ebi.eva.accession.release.batch.io.contig.ContigWriter; +import uk.ac.ebi.eva.accession.core.utils.PipelineTemporaryFolderUtil; import java.io.BufferedReader; import java.io.File; @@ -32,7 +30,8 @@ import java.util.List; import static junit.framework.TestCase.assertTrue; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; public class ContigWriterTest { @@ -60,28 +59,27 @@ public class ContigWriterTest { private File output; - @Rule - public TemporaryFolder temporaryFolderRule = new TemporaryFolder(); + public PipelineTemporaryFolderUtil temporaryFolderUtil = new PipelineTemporaryFolderUtil(); - @Before + @BeforeEach public void setUp() throws Exception { - output = temporaryFolderRule.newFile(); + output = temporaryFolderUtil.newFile(); } @Test public void write() throws Exception { ContigMapping contigMapping = new ContigMapping(Arrays.asList( new ContigSynonyms(SEQUENCE_NAME_1, "assembled-molecule", "1", GENBANK_ACCESSION_1, REFSEQ_ACCESSION_1, - "ucsc1", true), + "ucsc1", true), new ContigSynonyms(SEQUENCE_NAME_2, "assembled-molecule", "2", GENBANK_ACCESSION_2, REFSEQ_ACCESSION_2, - "ucsc2", false), + "ucsc2", false), new ContigSynonyms(SEQUENCE_NAME_3, "assembled-molecule", "3", GENBANK_ACCESSION_3, "na", "ucsc3", - false))); + false))); ContigWriter contigWriter = new ContigWriter(output, contigMapping); contigWriter.open(null); List contigs = Arrays.asList(GENBANK_ACCESSION_1, GENBANK_ACCESSION_2, GENBANK_ACCESSION_3); - contigWriter.write(contigs); + contigWriter.write(new Chunk<>(contigs)); contigWriter.close(); assertEquals(contigs.size(), numberOfLines(output)); @@ -110,16 +108,16 @@ private void assertContigFileContent(File file, List expectedLines) thro public void useSequenceNameIfContigIsRefSeqAccession() throws IOException { ContigMapping contigMapping = new ContigMapping(Arrays.asList( new ContigSynonyms(SEQUENCE_NAME_1, "assembled-molecule", "1", GENBANK_ACCESSION_1, REFSEQ_ACCESSION_1, - "ucsc1", true), + "ucsc1", true), new ContigSynonyms(SEQUENCE_NAME_2, "assembled-molecule", "2", GENBANK_ACCESSION_2, REFSEQ_ACCESSION_2, - "ucsc2", false), + "ucsc2", false), new ContigSynonyms(SEQUENCE_NAME_3, "assembled-molecule", "3", GENBANK_ACCESSION_3, "na", "ucsc3", - false))); + false))); ContigWriter contigWriter = new ContigWriter(output, contigMapping); contigWriter.open(null); List contigs = Arrays.asList(REFSEQ_ACCESSION_1, REFSEQ_ACCESSION_2, GENBANK_ACCESSION_3); - contigWriter.write(contigs); + contigWriter.write(new Chunk<>(contigs)); contigWriter.close(); assertEquals(contigs.size(), numberOfLines(output)); @@ -131,64 +129,64 @@ public void useSequenceNameIfContigIsRefSeqAccession() throws IOException { /** * This will happen a contig is null */ - @Test(expected = IllegalArgumentException.class) + @Test public void throwExceptionIfNullContig() { ContigMapping contigMapping = new ContigMapping(Arrays.asList( new ContigSynonyms(SEQUENCE_NAME_1, "assembled-molecule", "1", GENBANK_ACCESSION_1, REFSEQ_ACCESSION_1, - "ucsc1", true))); + "ucsc1", true))); ContigWriter contigWriter = new ContigWriter(output, contigMapping); contigWriter.open(null); String nullGenbankAccession = null; - contigWriter.write(Arrays.asList(GENBANK_ACCESSION_1, nullGenbankAccession)); + assertThrows(IllegalArgumentException.class, () -> contigWriter.write(Chunk.of(GENBANK_ACCESSION_1, nullGenbankAccession))); contigWriter.close(); } /** * This will happen a contig is empty */ - @Test(expected = IllegalArgumentException.class) + @Test public void throwExceptionIfEmptyContig() { ContigMapping contigMapping = new ContigMapping(Arrays.asList( new ContigSynonyms(SEQUENCE_NAME_1, "assembled-molecule", "1", GENBANK_ACCESSION_1, REFSEQ_ACCESSION_1, - "ucsc1", true))); + "ucsc1", true))); ContigWriter contigWriter = new ContigWriter(output, contigMapping); contigWriter.open(null); String emptyGenbankAccession = ""; - contigWriter.write(Arrays.asList(GENBANK_ACCESSION_1, emptyGenbankAccession)); + assertThrows(IllegalArgumentException.class, () -> contigWriter.write(Chunk.of(GENBANK_ACCESSION_1, emptyGenbankAccession))); contigWriter.close(); } /** * This will happen if the assembly report does not have a value for sequence name in at least one row */ - @Test(expected = IllegalArgumentException.class) + @Test public void throwExceptionIfEmptySequenceName() { ContigMapping contigMapping = new ContigMapping(Arrays.asList( new ContigSynonyms(SEQUENCE_NAME_1, "assembled-molecule", "1", GENBANK_ACCESSION_1, REFSEQ_ACCESSION_1, - "ucsc1", true), + "ucsc1", true), new ContigSynonyms(EMPTY_STRING, "assembled-molecule", "4", GENBANK_ACCESSION_EMPTY_SEQUENCE_NAME, "na", - "ucsc4", false))); + "ucsc4", false))); ContigWriter contigWriter = new ContigWriter(output, contigMapping); contigWriter.open(null); - contigWriter.write(Arrays.asList(GENBANK_ACCESSION_1, GENBANK_ACCESSION_EMPTY_SEQUENCE_NAME)); + assertThrows(IllegalArgumentException.class, () -> contigWriter.write(Chunk.of(GENBANK_ACCESSION_1, GENBANK_ACCESSION_EMPTY_SEQUENCE_NAME))); contigWriter.close(); } /** * This will happen if there is any contig in mongo that is not in the assembly report */ - @Test(expected = IllegalArgumentException.class) + @Test public void throwExceptionIfContigNotInAssemblyReport() { ContigMapping contigMapping = new ContigMapping(Arrays.asList( new ContigSynonyms(SEQUENCE_NAME_1, "assembled-molecule", "1", GENBANK_ACCESSION_1, REFSEQ_ACCESSION_1, - "ucsc1", true))); + "ucsc1", true))); ContigWriter contigWriter = new ContigWriter(output, contigMapping); contigWriter.open(null); - contigWriter.write(Arrays.asList(GENBANK_ACCESSION_1, GENBANK_ACCESSION_NOT_IN_ASSEMBLY_REPORT)); + assertThrows(IllegalArgumentException.class, () -> contigWriter.write(Chunk.of(GENBANK_ACCESSION_1, GENBANK_ACCESSION_NOT_IN_ASSEMBLY_REPORT))); contigWriter.close(); } } \ No newline at end of file diff --git a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/deprecated/DeprecatedVariantAccessionWriterTest.java b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/deprecated/DeprecatedVariantAccessionWriterTest.java index 690353127..f22a68e98 100644 --- a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/deprecated/DeprecatedVariantAccessionWriterTest.java +++ b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/deprecated/DeprecatedVariantAccessionWriterTest.java @@ -15,31 +15,28 @@ */ package uk.ac.ebi.eva.accession.release.batch.io.deprecated; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; - +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.batch.item.Chunk; +import uk.ac.ebi.eva.accession.core.utils.PipelineTemporaryFolderUtil; import uk.ac.ebi.eva.commons.core.models.pipeline.Variant; import java.io.BufferedReader; import java.io.File; import java.io.FileReader; import java.io.IOException; -import java.util.Arrays; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; public class DeprecatedVariantAccessionWriterTest { private DeprecatedVariantAccessionWriter deprecatedVariantAccessionWriter; - @Rule - public TemporaryFolder temporaryFolderRule = new TemporaryFolder(); + public PipelineTemporaryFolderUtil temporaryFolderUtil = new PipelineTemporaryFolderUtil(); - @Before + @BeforeEach public void setUp() throws Exception { - File output = temporaryFolderRule.newFile(); + File output = temporaryFolderUtil.newFile(); deprecatedVariantAccessionWriter = new DeprecatedVariantAccessionWriter(output.toPath()); } @@ -53,7 +50,7 @@ public void write() throws Exception { variant3.setMainId("rs3"); deprecatedVariantAccessionWriter.open(null); - deprecatedVariantAccessionWriter.write(Arrays.asList(variant1, variant2, variant3)); + deprecatedVariantAccessionWriter.write(Chunk.of(variant1, variant2, variant3)); deprecatedVariantAccessionWriter.close(); assertEquals(3, numberOfLines(deprecatedVariantAccessionWriter.getOutput())); diff --git a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/deprecated/DeprecatedVariantMongoReaderTest.java b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/deprecated/DeprecatedVariantMongoReaderTest.java index b4f902350..75adc3d34 100644 --- a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/deprecated/DeprecatedVariantMongoReaderTest.java +++ b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/deprecated/DeprecatedVariantMongoReaderTest.java @@ -16,21 +16,17 @@ package uk.ac.ebi.eva.accession.release.batch.io.deprecated; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; -import com.mongodb.MongoClient; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; +import com.mongodb.client.MongoClient; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.batch.item.ExecutionContext; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.ApplicationContext; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.ampt2d.commons.accession.core.models.EventType; import uk.ac.ebi.ampt2d.commons.accession.hashing.SHA1HashingFunction; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.MongoConfiguration; @@ -44,9 +40,9 @@ import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpSubmittedVariantOperationEntity; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; import uk.ac.ebi.eva.accession.core.summary.ClusteredVariantSummaryFunction; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.release.collectionNames.DbsnpCollectionNames; import uk.ac.ebi.eva.accession.release.test.configuration.MongoTestConfiguration; -import uk.ac.ebi.eva.accession.release.test.rule.FixSpringMongoDbRule; import uk.ac.ebi.eva.commons.core.models.pipeline.Variant; import java.util.ArrayList; @@ -57,13 +53,13 @@ import java.util.stream.Collectors; import java.util.stream.Stream; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; -@RunWith(SpringRunner.class) +@ExtendWith(SpringExtension.class) @TestPropertySource("classpath:application.properties") @ContextConfiguration(classes = {MongoConfiguration.class, MongoTestConfiguration.class}) -public class DeprecatedVariantMongoReaderTest { +public class DeprecatedVariantMongoReaderTest extends MongoTestContainerHelper { private static final String TEST_DB = "test-db"; @@ -83,22 +79,14 @@ public class DeprecatedVariantMongoReaderTest { @Autowired private MongoTemplate mongoTemplate; - //Required by nosql-unit - @Autowired - private ApplicationContext applicationContext; - - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName(TEST_DB).build()); - private DeprecatedVariantMongoReader reader; - @Before + @BeforeEach public void setUp() { this.mongoTemplate.getDb().drop(); } - @After + @AfterEach public void tearDown() { this.mongoTemplate.getDb().drop(); this.reader.close(); @@ -128,22 +116,22 @@ public void testOnlySpecifiedTaxVariantsRead() { // create RS this.mongoTemplate.insert(Stream.of(ss1, ss2, ss3, ss4).map(ss -> { DbsnpSubmittedVariantOperationEntity dbsnpSvoeObj = new DbsnpSubmittedVariantOperationEntity(); - dbsnpSvoeObj.fill( EventType.UPDATED, ss.getAccession(), - "Declustered: None of the variant alleles match the reference allele.", - Arrays.asList(new DbsnpSubmittedVariantInactiveEntity(ss))); + dbsnpSvoeObj.fill(EventType.UPDATED, ss.getAccession(), + "Declustered: None of the variant alleles match the reference allele.", + Arrays.asList(new DbsnpSubmittedVariantInactiveEntity(ss))); return dbsnpSvoeObj; }).collect(Collectors.toList()), DbsnpSubmittedVariantOperationEntity.class); this.mongoTemplate.insert(Stream.of(rs1, rs2, rs3, rs4).map(rs -> { DbsnpClusteredVariantOperationEntity dbsnpCvoeObj = new DbsnpClusteredVariantOperationEntity(); - dbsnpCvoeObj.fill( EventType.DEPRECATED, rs.getAccession(), - "Clustered variant completely declustered", - Arrays.asList(new DbsnpClusteredVariantInactiveEntity(rs))); + dbsnpCvoeObj.fill(EventType.DEPRECATED, rs.getAccession(), + "Clustered variant completely declustered", + Arrays.asList(new DbsnpClusteredVariantInactiveEntity(rs))); return dbsnpCvoeObj; }).collect(Collectors.toList()), DbsnpClusteredVariantOperationEntity.class); // Test records returned when the mongo reader is given TAXONOMY_1 this.reader = new DeprecatedVariantMongoReader(ASSEMBLY, TAXONOMY_1, mongoClient, TEST_DB, CHUNK_SIZE, - new DbsnpCollectionNames()); + new DbsnpCollectionNames()); this.reader.open(new ExecutionContext()); List deprecatedVariants = this.readIntoList(); assertEquals(1, deprecatedVariants.size()); @@ -153,7 +141,7 @@ public void testOnlySpecifiedTaxVariantsRead() { // Note rs3 is returned even though in Mongo it was originally issued in TAXONOMY_1 // This is because there is a SS record in TAXONOMY_2 that was declustered from rs3 this.reader = new DeprecatedVariantMongoReader(ASSEMBLY, TAXONOMY_2, mongoClient, TEST_DB, CHUNK_SIZE, - new DbsnpCollectionNames()); + new DbsnpCollectionNames()); this.reader.open(new ExecutionContext()); deprecatedVariants = this.readIntoList(); assertEquals(2, deprecatedVariants.size()); @@ -171,22 +159,22 @@ public void testOnlySpecifiedTaxVariantsRead() { private DbsnpSubmittedVariantEntity createSS(String assembly, int taxonomy, Long ssAccession, Long rsAccession, Long start, String reference, String alternate) { return new DbsnpSubmittedVariantEntity(ssAccession, "hash" + ssAccession, assembly, taxonomy, - "PRJ1", "chr1", start, reference, alternate, rsAccession, false, false, - false, false, 1); + "PRJ1", "chr1", start, reference, alternate, rsAccession, false, false, + false, false, 1); } private DbsnpClusteredVariantEntity createRS(SubmittedVariantEntity sve, Integer alternateTaxonomy, Long start) { - Function hashingFunction = new ClusteredVariantSummaryFunction().andThen( + Function hashingFunction = new ClusteredVariantSummaryFunction().andThen( new SHA1HashingFunction()); - int taxonomyToUse = Objects.isNull(alternateTaxonomy)? sve.getTaxonomyAccession(): alternateTaxonomy; + int taxonomyToUse = Objects.isNull(alternateTaxonomy) ? sve.getTaxonomyAccession() : alternateTaxonomy; long startToUse = Objects.isNull(start) ? sve.getStart() : start; ClusteredVariant cv = new ClusteredVariant(sve.getReferenceSequenceAccession(), taxonomyToUse, - sve.getContig(), - startToUse, - new Variant(sve.getContig(), sve.getStart(), sve.getStart(), - sve.getReferenceAllele(), - sve.getAlternateAllele()).getType(), - true, null); + sve.getContig(), + startToUse, + new Variant(sve.getContig(), sve.getStart(), sve.getStart(), + sve.getReferenceAllele(), + sve.getAlternateAllele()).getType(), + true, null); String hash = hashingFunction.apply(cv); return new DbsnpClusteredVariantEntity(sve.getClusteredVariantAccession(), hash, cv); } diff --git a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/merged/MergedVariantMongoReaderTest.java b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/merged/MergedVariantMongoReaderTest.java index da5f5e750..257d5bee5 100644 --- a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/merged/MergedVariantMongoReaderTest.java +++ b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/merged/MergedVariantMongoReaderTest.java @@ -16,23 +16,19 @@ package uk.ac.ebi.eva.accession.release.batch.io.merged; -import com.lordofthejars.nosqlunit.annotation.UsingDataSet; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; -import com.mongodb.MongoClient; -import org.junit.After; -import org.junit.Before; -import org.junit.Ignore; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; +import com.mongodb.client.MongoClient; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.batch.item.ExecutionContext; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.ApplicationContext; +import org.springframework.core.io.ResourceLoader; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.ampt2d.commons.accession.core.models.EventType; import uk.ac.ebi.ampt2d.commons.accession.hashing.SHA1HashingFunction; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.MongoConfiguration; @@ -47,10 +43,11 @@ import uk.ac.ebi.eva.accession.core.model.eva.ClusteredVariantEntity; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; import uk.ac.ebi.eva.accession.core.summary.ClusteredVariantSummaryFunction; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; +import uk.ac.ebi.eva.accession.core.utils.MongoTestDataLoader; import uk.ac.ebi.eva.accession.release.collectionNames.DbsnpCollectionNames; import uk.ac.ebi.eva.accession.release.collectionNames.EvaCollectionNames; import uk.ac.ebi.eva.accession.release.test.configuration.MongoTestConfiguration; -import uk.ac.ebi.eva.accession.release.test.rule.FixSpringMongoDbRule; import uk.ac.ebi.eva.commons.core.models.pipeline.Variant; import java.util.ArrayList; @@ -63,13 +60,11 @@ import java.util.stream.Collectors; import java.util.stream.Stream; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; -import static org.springframework.data.mongodb.core.query.Criteria.where; -import static org.springframework.data.mongodb.core.query.Query.query; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import static uk.ac.ebi.eva.accession.release.batch.io.active.AccessionedVariantMongoReader.CLUSTERED_VARIANT_VALIDATED_KEY; import static uk.ac.ebi.eva.accession.release.batch.io.active.AccessionedVariantMongoReader.REMAPPED_KEY; import static uk.ac.ebi.eva.accession.release.batch.io.active.AccessionedVariantMongoReader.SUBMITTED_VARIANT_VALIDATED_KEY; @@ -78,18 +73,10 @@ import static uk.ac.ebi.eva.accession.release.batch.io.merged.MergedVariantMongoReader.MERGED_INTO_KEY; import static uk.ac.ebi.eva.accession.release.batch.io.merged.MergedVariantMongoReader.SUPPORTED_BY_EVIDENCE_KEY; -@RunWith(SpringRunner.class) +@ExtendWith(SpringExtension.class) @TestPropertySource("classpath:application.properties") -@UsingDataSet(locations = { - "/test-data/dbsnpClusteredVariantOperationEntity.json", - "/test-data/clusteredVariantOperationEntity.json", - "/test-data/dbsnpSubmittedVariantOperationEntity.json", - "/test-data/submittedVariantOperationEntity.json", // includes 1 update involving dbSNP RS merge - "/test-data/dbsnpClusteredVariantEntity.json", - "/test-data/clusteredVariantEntity.json" // includes 1 RS that was a merge target for a dbSNP RS -}) @ContextConfiguration(classes = {MongoConfiguration.class, MongoTestConfiguration.class}) -public class MergedVariantMongoReaderTest { +public class MergedVariantMongoReaderTest extends MongoTestContainerHelper { private static final String TEST_DB = "test-db"; @@ -121,40 +108,42 @@ public class MergedVariantMongoReaderTest { @Autowired private MongoTemplate mongoTemplate; - //Required by nosql-unit @Autowired - private ApplicationContext applicationContext; - - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName(TEST_DB).build()); + private ResourceLoader resourceLoader; private MergedVariantMongoReader defaultReader; private ExecutionContext executionContext; - @Before + @BeforeEach public void setUp() throws Exception { + mongoTemplate.getDb().drop(); + + MongoTestDataLoader mongoTestDataLoader = new MongoTestDataLoader(mongoTemplate, resourceLoader); + mongoTestDataLoader.load("/test-data/dbsnpClusteredVariantOperationEntity.json"); + mongoTestDataLoader.load("/test-data/clusteredVariantOperationEntity.json"); + mongoTestDataLoader.load("/test-data/dbsnpSubmittedVariantOperationEntity.json"); + mongoTestDataLoader.load("/test-data/submittedVariantOperationEntity.json"); + mongoTestDataLoader.load("/test-data/dbsnpClusteredVariantEntity.json"); + mongoTestDataLoader.load("/test-data/clusteredVariantEntity.json"); + executionContext = new ExecutionContext(); defaultReader = new MergedVariantMongoReader(ASSEMBLY, TAXONOMY, mongoClient, TEST_DB, CHUNK_SIZE, - new DbsnpCollectionNames()); + new DbsnpCollectionNames()); } - @After - public void tearDown() throws Exception { - this.mongoTemplate.findAllAndRemove(query(where("inactiveObjects.tax").in(Arrays.asList(TAX1, TAX2))), - DbsnpSubmittedVariantOperationEntity.class); - this.mongoTemplate.findAllAndRemove(query(where("inactiveObjects.tax").in(Arrays.asList(TAX1, TAX2))), - DbsnpClusteredVariantOperationEntity.class); + @AfterEach + public void tearDown() { + mongoTemplate.getDb().drop(); } @Test - public void basicRead() throws Exception { + public void basicRead() { Map variants = readIntoMap(defaultReader); assertEquals(EXPECTED_MERGED_VARIANTS, variants.size()); } - private Map readIntoMap(MergedVariantMongoReader reader) throws Exception { + private Map readIntoMap(MergedVariantMongoReader reader) { reader.open(executionContext); Map allVariants = new HashMap<>(); List variants; @@ -173,23 +162,23 @@ private String getStringId(Variant variant) { } @Test - public void checkMergedInto() throws Exception { + public void checkMergedInto() { Map variants = readIntoMap(defaultReader); assertEquals(EXPECTED_MERGED_VARIANTS, variants.size()); assertTrue(variants.get(ID_1_A) - .getSourceEntries() - .stream() - .allMatch(e -> ID_1_MERGED_INTO.equals(e.getAttribute(MERGED_INTO_KEY)))); + .getSourceEntries() + .stream() + .allMatch(e -> ID_1_MERGED_INTO.equals(e.getAttribute(MERGED_INTO_KEY)))); assertTrue(variants.get(ID_1_T) - .getSourceEntries() - .stream() - .allMatch(e -> ID_1_MERGED_INTO.equals(e.getAttribute(MERGED_INTO_KEY)))); + .getSourceEntries() + .stream() + .allMatch(e -> ID_1_MERGED_INTO.equals(e.getAttribute(MERGED_INTO_KEY)))); } @Test - public void checkAlleles() throws Exception { + public void checkAlleles() { Map variants = readIntoMap(defaultReader); assertEquals(EXPECTED_MERGED_VARIANTS, variants.size()); @@ -204,58 +193,58 @@ public void checkAlleles() throws Exception { } @Test - public void includeValidatedFlag() throws Exception { + public void includeValidatedFlag() { assertFlagEqualsInAllVariants(CLUSTERED_VARIANT_VALIDATED_KEY, false); assertFlagEqualsInAllVariants(SUBMITTED_VARIANT_VALIDATED_KEY, false); } - private void assertFlagEqualsInAllVariants(String key, boolean value) throws Exception { + private void assertFlagEqualsInAllVariants(String key, boolean value) { Map variants = readIntoMap(defaultReader); assertNotEquals(0, variants.size()); assertTrue(variants.values().stream() - .flatMap(v -> v.getSourceEntries().stream()) - .map(se -> se.getAttribute(key)) - .map(Boolean::new) - .allMatch(v -> v.equals(value))); + .flatMap(v -> v.getSourceEntries().stream()) + .map(se -> se.getAttribute(key)) + .map(Boolean::parseBoolean) + .allMatch(v -> v.equals(value))); } @Test - public void includeAssemblyMatchFlag() throws Exception { + public void includeAssemblyMatchFlag() { assertFlagEqualsInAllVariants(ASSEMBLY_MATCH_KEY, true); } @Test - public void includeAllelesMatchFlag() throws Exception { + public void includeAllelesMatchFlag() { assertFlagEqualsInAllVariants(ALLELES_MATCH_KEY, true); } @Test - public void includeEvidenceFlag() throws Exception { + public void includeEvidenceFlag() { assertFlagEqualsInAllVariants(SUPPORTED_BY_EVIDENCE_KEY, false); } @Test - public void includeRemappedFlag() throws Exception { + public void includeRemappedFlag() { MergedVariantMongoReader evaReader = new MergedVariantMongoReader(ASSEMBLY, TAXONOMY, mongoClient, TEST_DB, - CHUNK_SIZE, new EvaCollectionNames()); + CHUNK_SIZE, new EvaCollectionNames()); Map variants = readIntoMap(evaReader); assertNotEquals(0, variants.size()); List rsToLookFor = variants.values().stream().filter(v -> v.getMainId().equals("rs3000000010")) - .collect(Collectors.toList()); + .collect(Collectors.toList()); assertNotEquals(0, rsToLookFor.size()); // Both the SS involved in SVOE/dbsnpSVOE operations has the "remappedFrom" attribute assertTrue(rsToLookFor.stream().flatMap(v -> v.getSourceEntries().stream()) - .map(se -> se.getAttribute(REMAPPED_KEY)) - .map(Boolean::new) - .allMatch(v -> v.equals(true))); + .map(se -> se.getAttribute(REMAPPED_KEY)) + .map(Boolean::parseBoolean) + .allMatch(v -> v.equals(true))); rsToLookFor = variants.values().stream().filter(v -> v.getMainId().equals("rs3000000020")) - .collect(Collectors.toList()); + .collect(Collectors.toList()); assertNotEquals(0, rsToLookFor.size()); // Only one of the SS involved in SVOE/dbsnpSVOE operations has the "remappedFrom" attribute assertTrue(rsToLookFor.stream().flatMap(v -> v.getSourceEntries().stream()) - .map(se -> se.getAttribute(REMAPPED_KEY)) - .map(Boolean::new) - .allMatch(v -> v.equals(false))); + .map(se -> se.getAttribute(REMAPPED_KEY)) + .map(Boolean::parseBoolean) + .allMatch(v -> v.equals(false))); } /** @@ -265,39 +254,39 @@ public void includeRemappedFlag() throws Exception { * - 1 Update operation in dbsnpSubmittedVariantOperationEntity for ss1986084768 (Original rs881301177 was merged * into rs80393223) * - 1 Merge operation in dbsnpSubmittedVariantOperationEntity for ss1986084768 - * + *

* Even though the rs80393223 was involved in a merge operation it doesn't mean all of its associated variants were * also involved in that merge. Hence we should only list the variants that were updated, merge operations of * submitted variants should be ignored. */ @Test - public void includeOnlyMergedVariants() throws Exception { + public void includeOnlyMergedVariants() { MergedVariantMongoReader reader = new MergedVariantMongoReader("GCA_000001215.4", 7227, - mongoClient, TEST_DB, - CHUNK_SIZE, new DbsnpCollectionNames()); + mongoClient, TEST_DB, + CHUNK_SIZE, new DbsnpCollectionNames()); Map allVariants = readIntoMap(reader); assertEquals(1, allVariants.size()); assertEquals("rs881301177", allVariants.get("AE013599.5_7680720_T_").getMainId()); assertEquals("rs80393223", allVariants.get("AE013599.5_7680720_T_").getSourceEntries().iterator().next() - .getAttributes().get("CURR")); + .getAttributes().get("CURR")); } /** * This test will use a different defaultReader for assembly GCA_000002305.1 to evaluate this specific scenario: * - 2 Merge operations for Clustered Variants with the same accession (rs69314228) and mergeInto (rs68736359) but - * different chromosome or/and start + * different chromosome or/and start * - 2 Update (merge RS) operations for Submitted Variants * - 1 Update (decluster) operation for Submitted Variants - * + *

* For every Clustered Variant Operation there will be N Submitted Variant Operations but only the merged * operations with the same chromosome and start must be taking into account to build the merged VCF file. */ @Test - public void includeOnlyMergedVariantsWithSameChrAndSameStartInRsAndSs() throws Exception { + public void includeOnlyMergedVariantsWithSameChrAndSameStartInRsAndSs() { MergedVariantMongoReader reader = new MergedVariantMongoReader("GCA_000002305.1", 9796, - mongoClient, TEST_DB, - CHUNK_SIZE, new DbsnpCollectionNames()); + mongoClient, TEST_DB, + CHUNK_SIZE, new DbsnpCollectionNames()); Map allVariants = readIntoMap(reader); assertEquals(2, allVariants.size()); @@ -313,18 +302,18 @@ public void includeOnlyMergedVariantsWithSameChrAndSameStartInRsAndSs() throws E * This test will use a different defaultReader for assembly GCA_000001635.5 to evaluate this specific scenario: * - One merge operation for clustered variants (rs258660893 merged into rs244942339) * - One update operations for submitted variants but the reason is a DECLUSTER - * + *

* This happens because the decluster operations are created in the processor and the merge operations in the writer. - * + *

* So we will have some situations where there is a merge operation but the merged clustered variant does not have * any submitted variants associated because they all have been declustered (rs = null). Hence no update operation * to indicate the merge is created for submitted variants. */ @Test - public void noExceptionIfSsWereDeclustered() throws Exception { + public void noExceptionIfSsWereDeclustered() { MergedVariantMongoReader reader = new MergedVariantMongoReader("GCA_000001635.5", 10090, - mongoClient, TEST_DB, - CHUNK_SIZE, new DbsnpCollectionNames()); + mongoClient, TEST_DB, + CHUNK_SIZE, new DbsnpCollectionNames()); Map allVariants = readIntoMap(reader); assertEquals(0, allVariants.size()); } @@ -333,14 +322,15 @@ public void noExceptionIfSsWereDeclustered() throws Exception { * This test will use a different defaultReader for assembly GCA_000181335.3 to evaluate this specific scenario: * - One merge operation for clustered variants (rs782965190 merged into rs43955718) * - No submitted variants operations associated to rs782965190 - * + *

* Given that there are not merge nor decluster operations for any submitted variant associated this should * not return any records */ - public void exceptionIfRsMergedHasNoSsMergeOperations() throws Exception { + @Test + public void exceptionIfRsMergedHasNoSsMergeOperations() { MergedVariantMongoReader reader = new MergedVariantMongoReader("GCA_000181335.3", 9685, - mongoClient, TEST_DB, - CHUNK_SIZE, new DbsnpCollectionNames()); + mongoClient, TEST_DB, + CHUNK_SIZE, new DbsnpCollectionNames()); assertEquals(0, readIntoMap(reader).size()); } @@ -349,14 +339,14 @@ public void exceptionIfRsMergedHasNoSsMergeOperations() throws Exception { * - Two merge operation for clustered variants * - Two corresponding operations for its submitted variants * - Only one of the clustered variants is active (Present in dbsnpClusteredVariantEntity collection) - * + *

* Only the clustered operation associated to the active clustered variant should be returned */ @Test public void excludeMergedIntoADeprecatedRs() throws Exception { MergedVariantMongoReader reader = new MergedVariantMongoReader("GCA_000004515.3", 3847, - mongoClient, TEST_DB, - CHUNK_SIZE, new DbsnpCollectionNames()); + mongoClient, TEST_DB, + CHUNK_SIZE, new DbsnpCollectionNames()); Map allVariants = readIntoMap(reader); assertEquals(1, allVariants.size()); assertNotNull(allVariants.get("CM000851.2_2715437_G_A")); @@ -368,24 +358,24 @@ public void excludeMergedIntoADeprecatedRs() throws Exception { * and that RS ID in one location can be deprecated but active in the other one. In that case we will count the * RS ID as active because it will be present in the dbsnpClusteredVariantEntity collection for at least one * location. - * + *

* This test will use a different defaultReader for assembly GCA_000001111.1 to evaluate this specific scenario: * - Two merge operations for clustered variants to the same RS ID (rs2222->rs1111 and rs3333->rs1111). Note that * rs1111 is mapped to multiple locations (start:100 and start:200) * - Two submitted variant operations indicating the merged operations of its RS ID * - Only one clustered variant in the active collection (rs1111 start:100). This means that rs1111 start:200 has * been deprecated - * + *

* Even though the rs1111 is only active with start 100, the merge reader will also include rs1111 as * "current RS ID" for the line of rs2222 with start 200 in the merged VCF. */ - @Ignore("This test is ignored because it would fail with the current implementation but we need to be aware of" + + @Disabled("This test is ignored because it would fail with the current implementation but we need to be aware of" + "this situation (More details in javadoc)") @Test - public void rsMappedToDifferentLocationsOneDeprecated() throws Exception { + public void rsMappedToDifferentLocationsOneDeprecated() { MergedVariantMongoReader reader = new MergedVariantMongoReader("GCA_000001111.1", 1111, - mongoClient, TEST_DB, - CHUNK_SIZE, new DbsnpCollectionNames()); + mongoClient, TEST_DB, + CHUNK_SIZE, new DbsnpCollectionNames()); Map allVariants = readIntoMap(reader); //Should return only RS ID with start:100 @@ -404,14 +394,14 @@ public void rsMappedToDifferentLocationsOneDeprecated() throws Exception { * This test will use a different defaultReader for assembly GCA_000001635.4 to evaluate this specific scenario: * - One merge operation for clustered variants (rs258660892 merged into rs244942338) * - One update operation for submitted variants but in the EVA collection (clustered under a dbSNP RS). - * + *

* In this case we should not throw an exception but find the variant in the other collection. */ @Test - public void noExceptionIfSsFromDifferentCollectionUpdated() throws Exception { + public void noExceptionIfSsFromDifferentCollectionUpdated() { MergedVariantMongoReader reader = new MergedVariantMongoReader("GCA_000001635.4", 10090, - mongoClient, TEST_DB, - CHUNK_SIZE, new DbsnpCollectionNames()); + mongoClient, TEST_DB, + CHUNK_SIZE, new DbsnpCollectionNames()); Map allVariants = readIntoMap(reader); assertEquals(1, allVariants.size()); } @@ -419,20 +409,20 @@ public void noExceptionIfSsFromDifferentCollectionUpdated() throws Exception { /** * This test will use a different defaultReader for assembly GCA_002863925.1 to evaluate this specific scenario: * - One merge operation for clustered variants (dbSNP rs3091764863 merged into EVA rs393745096) - * + *

* In this case we should successfully find the merged dbSNP clustered variant. */ @Test - public void rsMergedWithRsFromDifferentCollection() throws Exception { + public void rsMergedWithRsFromDifferentCollection() { MergedVariantMongoReader reader = new MergedVariantMongoReader("GCA_002863925.1", 9796, - mongoClient, TEST_DB, - CHUNK_SIZE, new DbsnpCollectionNames()); + mongoClient, TEST_DB, + CHUNK_SIZE, new DbsnpCollectionNames()); Map allVariants = readIntoMap(reader); assertEquals(1, allVariants.size()); } @Test - public void testOnlySpecifiedTaxVariantsRead() throws Exception { + public void testOnlySpecifiedTaxVariantsRead() { // See scenario here: https://docs.google.com/spreadsheets/d/12QJT4N0-UJGTv3BtVq_gyyrVzweXd5ev2WFlnP-4MW4/edit#rangeid=169520205 DbsnpSubmittedVariantEntity ss1 = createSS(ASSEMBLY, TAX1, 1L, 1L, 100L, "C", "A"); DbsnpSubmittedVariantEntity ss2 = createSS(ASSEMBLY, TAX2, 2L, 2L, 101L, "A", "T"); @@ -443,26 +433,26 @@ public void testOnlySpecifiedTaxVariantsRead() throws Exception { this.mongoTemplate.insert(Stream.of(ss1, ss2, ss3).map(ss -> { DbsnpSubmittedVariantOperationEntity dbsnpSvoeObj = new DbsnpSubmittedVariantOperationEntity(); dbsnpSvoeObj.fill(EventType.UPDATED, ss.getAccession(), - "Original RS was merged into another RS", - Arrays.asList(new DbsnpSubmittedVariantInactiveEntity(ss))); + "Original RS was merged into another RS", + Arrays.asList(new DbsnpSubmittedVariantInactiveEntity(ss))); return dbsnpSvoeObj; }).collect(Collectors.toList()), DbsnpSubmittedVariantOperationEntity.class); this.mongoTemplate.insert(Stream.of(rs1, rs2, rs3).map(rs -> { DbsnpClusteredVariantOperationEntity dbsnpCvoeObj = new DbsnpClusteredVariantOperationEntity(); - dbsnpCvoeObj.fill( EventType.MERGED, rs.getAccession(), rs.getAccession() + 10, - "Original RS was merged into another RS", - Arrays.asList(new DbsnpClusteredVariantInactiveEntity(rs))); + dbsnpCvoeObj.fill(EventType.MERGED, rs.getAccession(), rs.getAccession() + 10, + "Original RS was merged into another RS", + Arrays.asList(new DbsnpClusteredVariantInactiveEntity(rs))); return dbsnpCvoeObj; }).collect(Collectors.toList()), DbsnpClusteredVariantOperationEntity.class); // If we don't insert target CVEs of the merges, the merge reader will fail // because such merged RS will be categorized as merged-then-deprecated this.mongoTemplate.insert(Stream.of(rs1, rs2, rs3).map(rs -> new ClusteredVariantEntity( - rs.getAccession() + 10, rs.getHashedMessage(), rs.getModel())) - .collect(Collectors.toList()), DbsnpClusteredVariantEntity.class); + rs.getAccession() + 10, rs.getHashedMessage(), rs.getModel())) + .collect(Collectors.toList()), DbsnpClusteredVariantEntity.class); // Test records returned when the mongo reader is given TAX1 - MergedVariantMongoReader reader = new MergedVariantMongoReader(ASSEMBLY, TAX1, mongoClient, TEST_DB, - CHUNK_SIZE, new DbsnpCollectionNames()); + MergedVariantMongoReader reader = new MergedVariantMongoReader(ASSEMBLY, TAX1, mongoClient, TEST_DB, + CHUNK_SIZE, new DbsnpCollectionNames()); reader.open(new ExecutionContext()); List mergedVariants = new ArrayList<>(readIntoMap(reader).values()); assertEquals(1, mergedVariants.size()); @@ -472,7 +462,7 @@ public void testOnlySpecifiedTaxVariantsRead() throws Exception { // Note rs3 is returned even though in Mongo it was originally issued in TAX1 // This is because, before the merge, there was a SS record in TAX2 that was clustered under rs3 reader = new MergedVariantMongoReader(ASSEMBLY, TAX2, mongoClient, TEST_DB, CHUNK_SIZE, - new DbsnpCollectionNames()); + new DbsnpCollectionNames()); reader.open(new ExecutionContext()); mergedVariants = new ArrayList<>(readIntoMap(reader).values()); assertEquals(2, mergedVariants.size()); @@ -483,21 +473,21 @@ public void testOnlySpecifiedTaxVariantsRead() throws Exception { private DbsnpSubmittedVariantEntity createSS(String assembly, int taxonomy, Long ssAccession, Long rsAccession, Long start, String reference, String alternate) { return new DbsnpSubmittedVariantEntity(ssAccession, "hash" + ssAccession, assembly, taxonomy, - "PRJ1", "chr1", start, reference, alternate, rsAccession, false, false, - false, false, 1); + "PRJ1", "chr1", start, reference, alternate, rsAccession, false, false, + false, false, 1); } private DbsnpClusteredVariantEntity createRS(SubmittedVariantEntity sve, Integer alternateTaxonomy) { - Function hashingFunction = new ClusteredVariantSummaryFunction().andThen( + Function hashingFunction = new ClusteredVariantSummaryFunction().andThen( new SHA1HashingFunction()); - int taxonomyToUse = Objects.isNull(alternateTaxonomy)? sve.getTaxonomyAccession(): alternateTaxonomy; + int taxonomyToUse = Objects.isNull(alternateTaxonomy) ? sve.getTaxonomyAccession() : alternateTaxonomy; ClusteredVariant cv = new ClusteredVariant(sve.getReferenceSequenceAccession(), taxonomyToUse, - sve.getContig(), - sve.getStart(), - new Variant(sve.getContig(), sve.getStart(), sve.getStart(), - sve.getReferenceAllele(), - sve.getAlternateAllele()).getType(), - true, null); + sve.getContig(), + sve.getStart(), + new Variant(sve.getContig(), sve.getStart(), sve.getStart(), + sve.getReferenceAllele(), + sve.getAlternateAllele()).getType(), + true, null); String hash = hashingFunction.apply(cv); return new DbsnpClusteredVariantEntity(sve.getClusteredVariantAccession(), hash, cv); } diff --git a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/merged_deprecated/MergedDeprecatedVariantAccessionWriterTest.java b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/merged_deprecated/MergedDeprecatedVariantAccessionWriterTest.java index 432af7f40..b9193fad0 100644 --- a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/merged_deprecated/MergedDeprecatedVariantAccessionWriterTest.java +++ b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/merged_deprecated/MergedDeprecatedVariantAccessionWriterTest.java @@ -15,33 +15,31 @@ */ package uk.ac.ebi.eva.accession.release.batch.io.merged_deprecated; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.batch.item.Chunk; import uk.ac.ebi.ampt2d.commons.accession.core.models.EventType; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpClusteredVariantOperationEntity; +import uk.ac.ebi.eva.accession.core.utils.PipelineTemporaryFolderUtil; import java.io.BufferedReader; import java.io.File; import java.io.FileReader; import java.io.IOException; -import java.util.Arrays; import java.util.Optional; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; public class MergedDeprecatedVariantAccessionWriterTest { private MergedDeprecatedVariantAccessionWriter writer; - @Rule - public TemporaryFolder temporaryFolderRule = new TemporaryFolder(); + public PipelineTemporaryFolderUtil temporaryFolderUtil = new PipelineTemporaryFolderUtil(); - @Before + @BeforeEach public void setUp() throws Exception { - File output = temporaryFolderRule.newFile(); + File output = temporaryFolderUtil.newFile(); writer = new MergedDeprecatedVariantAccessionWriter(output.toPath()); } @@ -55,7 +53,7 @@ public void writeLines() throws Exception { variant3.fill(EventType.MERGED, 3L, 13L, "Reason", null); writer.open(null); - writer.write(Arrays.asList(variant1, variant2, variant3)); + writer.write(Chunk.of(variant1, variant2, variant3)); writer.close(); assertEquals(3, numberOfLines(writer.getOutput())); @@ -73,7 +71,7 @@ public void twoColumns() throws Exception { long accessionIdDestiny = 11L; variant1.fill(EventType.MERGED, accessionIdOrigin, accessionIdDestiny, "Reason", null); writer.open(null); - writer.write(Arrays.asList(variant1)); + writer.write(Chunk.of(variant1)); writer.close(); Optional line = new BufferedReader(new FileReader( diff --git a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/merged_deprecated/MergedDeprecatedVariantMongoReaderTest.java b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/merged_deprecated/MergedDeprecatedVariantMongoReaderTest.java index f151e0dfe..ef211280c 100644 --- a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/merged_deprecated/MergedDeprecatedVariantMongoReaderTest.java +++ b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/merged_deprecated/MergedDeprecatedVariantMongoReaderTest.java @@ -16,44 +16,35 @@ package uk.ac.ebi.eva.accession.release.batch.io.merged_deprecated; -import com.lordofthejars.nosqlunit.annotation.UsingDataSet; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; -import com.mongodb.MongoClient; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; +import com.mongodb.client.MongoClient; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.batch.item.ExecutionContext; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.ApplicationContext; +import org.springframework.core.io.ResourceLoader; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.ampt2d.commons.accession.core.models.EventType; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.MongoConfiguration; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpClusteredVariantOperationEntity; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; +import uk.ac.ebi.eva.accession.core.utils.MongoTestDataLoader; import uk.ac.ebi.eva.accession.release.collectionNames.DbsnpCollectionNames; import uk.ac.ebi.eva.accession.release.test.configuration.MongoTestConfiguration; -import uk.ac.ebi.eva.accession.release.test.rule.FixSpringMongoDbRule; import java.util.ArrayList; import java.util.List; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; -@RunWith(SpringRunner.class) +@ExtendWith(SpringExtension.class) @TestPropertySource("classpath:application.properties") -@UsingDataSet(locations = { - "/test-data/dbsnpClusteredVariantOperationEntity.json", - "/test-data/dbsnpSubmittedVariantOperationEntity.json", - "/test-data/dbsnpClusteredVariantEntity.json", - "/test-data/dbsnpSubmittedVariantEntity.json" -}) @ContextConfiguration(classes = {MongoConfiguration.class, MongoTestConfiguration.class}) -public class MergedDeprecatedVariantMongoReaderTest { +public class MergedDeprecatedVariantMongoReaderTest extends MongoTestContainerHelper { private static final String TEST_DB = "test-db"; @@ -66,30 +57,35 @@ public class MergedDeprecatedVariantMongoReaderTest { @Autowired private MongoTemplate mongoTemplate; - @Autowired - private MongoClient mongoClient; - //Required by nosql-unit @Autowired - private ApplicationContext applicationContext; + private ResourceLoader resourceLoader; - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName(TEST_DB).build()); + @Autowired + private MongoClient mongoClient; private MergedDeprecatedVariantMongoReader reader; - @Before + @BeforeEach public void setUp() { + mongoTemplate.getDb().drop(); + + MongoTestDataLoader mongoTestDataLoader = new MongoTestDataLoader(mongoTemplate, resourceLoader); + mongoTestDataLoader.load("/test-data/dbsnpClusteredVariantOperationEntity.json"); + mongoTestDataLoader.load("/test-data/dbsnpSubmittedVariantOperationEntity.json"); + mongoTestDataLoader.load("/test-data/dbsnpClusteredVariantEntity.json"); + mongoTestDataLoader.load("/test-data/dbsnpSubmittedVariantEntity.json"); + ExecutionContext executionContext = new ExecutionContext(); reader = new DbsnpMergedDeprecatedVariantMongoReader(ASSEMBLY, TAXONOMY, mongoClient, TEST_DB, - mongoTemplate.getConverter(), CHUNK_SIZE, - new DbsnpCollectionNames()); + mongoTemplate.getConverter(), CHUNK_SIZE, + new DbsnpCollectionNames()); reader.open(executionContext); } - @After + @AfterEach public void tearDown() { + mongoTemplate.getDb().drop(); reader.close(); } diff --git a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/processors/ContextNucleotideAdditionProcessorTest.java b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/processors/ContextNucleotideAdditionProcessorTest.java index f88ce0cfb..d304c7d00 100644 --- a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/processors/ContextNucleotideAdditionProcessorTest.java +++ b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/processors/ContextNucleotideAdditionProcessorTest.java @@ -17,13 +17,12 @@ */ package uk.ac.ebi.eva.accession.release.batch.processors; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; - +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import uk.ac.ebi.eva.accession.core.batch.io.FastaSynonymSequenceReader; import uk.ac.ebi.eva.accession.core.contig.ContigMapping; import uk.ac.ebi.eva.accession.core.contig.ContigSynonyms; -import uk.ac.ebi.eva.accession.core.batch.io.FastaSynonymSequenceReader; import uk.ac.ebi.eva.accession.core.exceptions.PositionOutsideOfContigException; import uk.ac.ebi.eva.commons.core.models.IVariant; import uk.ac.ebi.eva.commons.core.models.pipeline.Variant; @@ -33,9 +32,11 @@ import java.util.Arrays; import java.util.Collections; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.fail; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; public class ContextNucleotideAdditionProcessorTest { @@ -53,18 +54,18 @@ public class ContextNucleotideAdditionProcessorTest { private static ContextNucleotideAdditionProcessor contextNucleotideAdditionProcessor; - @BeforeClass + @BeforeAll public static void setUpClass() throws Exception { Path fastaPath = Paths.get("../eva-accession-core/src/test/resources/input-files/fasta/Gallus_gallus-5.0.test.fa"); ContigMapping contigMapping = new ContigMapping( Arrays.asList(new ContigSynonyms(CONTIG, "", "", "", "", "", true), - new ContigSynonyms(SCAFFOLD, "", "", SCAFFOLD_GENBANK_IN_FASTA, "", "", true), - new ContigSynonyms(MISSING_IN_FASTA, "", "", "", "", "", true))); + new ContigSynonyms(SCAFFOLD, "", "", SCAFFOLD_GENBANK_IN_FASTA, "", "", true), + new ContigSynonyms(MISSING_IN_FASTA, "", "", "", "", "", true))); fastaSynonymSequenceReader = new FastaSynonymSequenceReader(contigMapping, fastaPath); contextNucleotideAdditionProcessor = new ContextNucleotideAdditionProcessor(fastaSynonymSequenceReader); } - @AfterClass + @AfterAll public static void tearDownClass() throws Exception { fastaSynonymSequenceReader.close(); } @@ -198,15 +199,18 @@ public void addContextBaseUsingSynonymContig() throws Exception { assertEquals("TA", processedVariant.getAlternate()); } - @Test(expected = IllegalArgumentException.class) - public void contigNotFound() throws Exception { + @Test + void contigNotFound() { Variant variant1 = new Variant(MISSING_IN_FASTA, 10, 10, "", "A"); - try { - contextNucleotideAdditionProcessor.process(variant1); - } catch (PositionOutsideOfContigException wrongException) { - fail("The exception (" + wrongException.getClass().getSimpleName() - + ") is wrong because the variant doesn't have a position outside of chromosome. The correct " - + "exception should be that the contig is not present in the fasta"); + + Exception exception = assertThrows(Exception.class, () -> contextNucleotideAdditionProcessor.process(variant1)); + + if (exception instanceof PositionOutsideOfContigException) { + fail("The exception (" + exception.getClass().getSimpleName() + + ") is wrong because the variant doesn't have a position outside of chromosome. The correct " + + "exception should be that the contig is not present in the fasta"); } + + assertTrue(exception instanceof IllegalArgumentException); } } diff --git a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/processors/ExcludeInvalidVariantsProcessorTest.java b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/processors/ExcludeInvalidVariantsProcessorTest.java index a16b5cecf..4c6153bf6 100644 --- a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/processors/ExcludeInvalidVariantsProcessorTest.java +++ b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/processors/ExcludeInvalidVariantsProcessorTest.java @@ -16,16 +16,13 @@ package uk.ac.ebi.eva.accession.release.batch.processors; -import org.junit.BeforeClass; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; - +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; import uk.ac.ebi.eva.commons.core.models.pipeline.Variant; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; public class ExcludeInvalidVariantsProcessorTest { @@ -49,10 +46,7 @@ public class ExcludeInvalidVariantsProcessorTest { private static ExcludeInvalidVariantsProcessor processor; - @Rule - public ExpectedException thrown = ExpectedException.none(); - - @BeforeClass + @BeforeAll public static void setUp() { processor = new ExcludeInvalidVariantsProcessor(); } diff --git a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/processors/NamedVariantProcessorTest.java b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/processors/NamedVariantProcessorTest.java index 4e9a0d866..ab29a9a1c 100644 --- a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/processors/NamedVariantProcessorTest.java +++ b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/processors/NamedVariantProcessorTest.java @@ -16,7 +16,7 @@ package uk.ac.ebi.eva.accession.release.batch.processors; -import org.junit.Test; +import org.junit.jupiter.api.Test; import uk.ac.ebi.eva.commons.core.models.IVariant; import uk.ac.ebi.eva.commons.core.models.VariantType; @@ -24,8 +24,9 @@ import uk.ac.ebi.eva.commons.core.models.pipeline.Variant; import uk.ac.ebi.eva.commons.core.models.pipeline.VariantSourceEntry; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import static uk.ac.ebi.eva.accession.release.batch.io.VariantMongoAggregationReader.ALLELES_MATCH_KEY; import static uk.ac.ebi.eva.accession.release.batch.io.VariantMongoAggregationReader.ASSEMBLY_MATCH_KEY; import static uk.ac.ebi.eva.accession.release.batch.io.VariantMongoAggregationReader.CLUSTERED_VARIANT_VALIDATED_KEY; @@ -72,7 +73,7 @@ private void assertNamedVariant(String reference, String alternate) throws Excep private Variant buildVariant(String chr, int start, String reference, String alternate, String sequenceOntologyTerm, String... studies) { return buildVariant(chr, start, reference, alternate, sequenceOntologyTerm, false, false, true, true, true, - studies); + studies); } private Variant buildVariant(String chr, int start, String reference, String alternate, @@ -109,9 +110,9 @@ public void processNamedDeletionWithEmptyAlternate() throws Exception { assertNamedVariant("(1190 BP DEL)", ""); } - @Test(expected = IllegalArgumentException.class) - public void throwIfBothAllelesAreNamed() throws Exception { - assertNamedVariant("(1190 BP DEL)", "(1190 BP DEL)"); + @Test + public void throwIfBothAllelesAreNamed() { + assertThrows(IllegalArgumentException.class, () -> assertNamedVariant("(1190 BP DEL)", "(1190 BP DEL)")); } @Test @@ -153,7 +154,7 @@ public void swapSymbolicReference() throws Exception { } /** - * Taken from EVA-1220. Some of these won't even go through the release in VCF format, but test them just in case + * Taken from EVA-1220. Some of these won't even go through the release in VCF format, but test them just in case */ @Test public void handleStrangeAlleles() throws Exception { diff --git a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/processors/VariantToVariantContextProcessorTest.java b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/processors/VariantToVariantContextProcessorTest.java index d8f17bd0e..210aa9fa7 100644 --- a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/processors/VariantToVariantContextProcessorTest.java +++ b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/processors/VariantToVariantContextProcessorTest.java @@ -19,11 +19,8 @@ import htsjdk.variant.variantcontext.Allele; import htsjdk.variant.variantcontext.VariantContext; import org.assertj.core.util.Sets; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; - +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import uk.ac.ebi.eva.accession.core.contig.ContigMapping; import uk.ac.ebi.eva.accession.core.contig.ContigNaming; import uk.ac.ebi.eva.accession.core.contig.ContigSynonyms; @@ -32,8 +29,9 @@ import java.util.Collections; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; public class VariantToVariantContextProcessorTest { @@ -61,10 +59,7 @@ public class VariantToVariantContextProcessorTest { private VariantToVariantContextProcessor variantConverter; - @Rule - public ExpectedException expectedException = ExpectedException.none(); - - @Before + @BeforeEach public void setUp() throws Exception { ContigMapping contigMapping = new ContigMapping(Collections.singletonList( new ContigSynonyms(SEQUENCE_NAME_1, "A", "A", GENBANK_ACCESSION_1, "A", "A", true))); @@ -72,7 +67,7 @@ public void setUp() throws Exception { } @Test - public void singleStudySNV() throws Exception { + public void singleStudySNV() { Variant variant = buildVariant(GENBANK_ACCESSION_1, 1000, "C", "A", SNP_SEQUENCE_ONTOLOGY, STUDY_1); VariantContext variantContext = variantConverter.process(variant); assertVariantContext(variantContext, SEQUENCE_NAME_1, 1000, 1000, ID, "C", "A", SNP_SEQUENCE_ONTOLOGY, STUDY_1); @@ -100,7 +95,7 @@ private void assertVariantContext(VariantContext variantContext, String expected assertEquals(expectedEnd, variantContext.getEnd()); assertEquals(Allele.create(expectedReference, true), variantContext.getReference()); assertEquals(Collections.singletonList(Allele.create(expectedAlternate, false)), - variantContext.getAlternateAlleles()); + variantContext.getAlternateAlleles()); assertEquals(expectedId, variantContext.getID()); assertTrue(variantContext.getFilters().isEmpty()); assertEquals(2, variantContext.getCommonInfo().getAttributes().size()); @@ -115,61 +110,59 @@ private void assertVariantContext(VariantContext variantContext, String expected } @Test - public void throwsIfAllelesAreEmpty() throws Exception { + public void throwsIfAllelesAreEmpty() { Variant variant = buildVariant(GENBANK_ACCESSION_1, 1100, "", "G", SNP_SEQUENCE_ONTOLOGY, STUDY_1); - expectedException.expect(IllegalArgumentException.class); - variantConverter.process(variant); + assertThrows(IllegalArgumentException.class, () -> variantConverter.process(variant)); } @Test - public void singleStudySingleNucleotideInsertion() throws Exception { + public void singleStudySingleNucleotideInsertion() { Variant variant = buildVariant(GENBANK_ACCESSION_1, 1100, "T", "TG", SNP_SEQUENCE_ONTOLOGY, STUDY_1); VariantContext variantContext = variantConverter.process(variant); assertVariantContext(variantContext, SEQUENCE_NAME_1, 1100, 1100, ID, "T", "TG", SNP_SEQUENCE_ONTOLOGY, - STUDY_1); + STUDY_1); } @Test - public void singleStudySeveralNucleotidesInsertion() throws Exception { + public void singleStudySeveralNucleotidesInsertion() { Variant variant = buildVariant(GENBANK_ACCESSION_1, 1100, "T", "TGA", INSERTION_SEQUENCE_ONTOLOGY, STUDY_1); VariantContext variantContext = variantConverter.process(variant); assertVariantContext(variantContext, SEQUENCE_NAME_1, 1100, 1100, ID, "T", "TGA", INSERTION_SEQUENCE_ONTOLOGY, - STUDY_1); + STUDY_1); } @Test - public void singleStudySingleNucleotideDeletion() throws Exception { + public void singleStudySingleNucleotideDeletion() { Variant variant = buildVariant(GENBANK_ACCESSION_1, 1100, "TA", "T", DELETION_SEQUENCE_ONTOLOGY, STUDY_1); VariantContext variantContext = variantConverter.process(variant); assertVariantContext(variantContext, SEQUENCE_NAME_1, 1100, 1101, ID, "TA", "T", DELETION_SEQUENCE_ONTOLOGY, - STUDY_1); + STUDY_1); } @Test - public void singleStudySeveralNucleotidesDeletion() throws Exception { + public void singleStudySeveralNucleotidesDeletion() { Variant variant = buildVariant(GENBANK_ACCESSION_1, 1100, "TAG", "T", DELETION_SEQUENCE_ONTOLOGY, STUDY_1); VariantContext variantContext = variantConverter.process(variant); assertVariantContext(variantContext, SEQUENCE_NAME_1, 1100, 1102, ID, "TAG", "T", DELETION_SEQUENCE_ONTOLOGY, - STUDY_1); + STUDY_1); } @Test - public void singleStudyMultiAllelicVariant() throws Exception { + public void singleStudyMultiAllelicVariant() { Variant variant = buildVariant(GENBANK_ACCESSION_1, 1100, "C", "A,T", SNP_SEQUENCE_ONTOLOGY, STUDY_1); - expectedException.expect(IllegalArgumentException.class); - variantConverter.process(variant); + assertThrows(IllegalArgumentException.class, () -> variantConverter.process(variant)); } @Test - public void singleNucleotideInsertionInPosition1() throws Exception { + public void singleNucleotideInsertionInPosition1() { Variant variant = buildVariant(GENBANK_ACCESSION_1, 1, "A", "TA", INSERTION_SEQUENCE_ONTOLOGY, STUDY_1); VariantContext variantContext = variantConverter.process(variant); assertVariantContext(variantContext, SEQUENCE_NAME_1, 1, 1, ID, "A", "TA", INSERTION_SEQUENCE_ONTOLOGY, - STUDY_1); + STUDY_1); } @Test - public void singleNucleotideDeletionInPosition1() throws Exception { + public void singleNucleotideDeletionInPosition1() { Variant variant = buildVariant(GENBANK_ACCESSION_1, 1, "AT", "T", DELETION_SEQUENCE_ONTOLOGY, STUDY_1); VariantContext variantContext = variantConverter.process(variant); assertVariantContext(variantContext, SEQUENCE_NAME_1, 1, 2, ID, "AT", "T", DELETION_SEQUENCE_ONTOLOGY, STUDY_1); @@ -177,23 +170,23 @@ public void singleNucleotideDeletionInPosition1() throws Exception { @Test - public void severalNucleotidesInsertionInPosition1() throws Exception { + public void severalNucleotidesInsertionInPosition1() { Variant variant = buildVariant(GENBANK_ACCESSION_1, 1, "A", "GGTA", INSERTION_SEQUENCE_ONTOLOGY, STUDY_1); VariantContext variantContext = variantConverter.process(variant); assertVariantContext(variantContext, SEQUENCE_NAME_1, 1, 1, ID, "A", "GGTA", INSERTION_SEQUENCE_ONTOLOGY, - STUDY_1); + STUDY_1); } @Test - public void severalNucleotidesDeletionInPosition1() throws Exception { + public void severalNucleotidesDeletionInPosition1() { Variant variant = buildVariant(GENBANK_ACCESSION_1, 1, "ATTG", "G", DELETION_SEQUENCE_ONTOLOGY, STUDY_1); VariantContext variantContext = variantConverter.process(variant); assertVariantContext(variantContext, SEQUENCE_NAME_1, 1, 4, ID, "ATTG", "G", DELETION_SEQUENCE_ONTOLOGY, - STUDY_1); + STUDY_1); } @Test - public void twoStudiesSingleVariant() throws Exception { + public void twoStudiesSingleVariant() { Variant variant = buildVariant(GENBANK_ACCESSION_1, 1000, "T", "G", SNP_SEQUENCE_ONTOLOGY, STUDY_1, STUDY_2); // process variant @@ -201,7 +194,7 @@ public void twoStudiesSingleVariant() throws Exception { // check processed variant assertVariantContext(variantContext, SEQUENCE_NAME_1, 1000, 1000, ID, "T", "G", SNP_SEQUENCE_ONTOLOGY, STUDY_1, - STUDY_2); + STUDY_2); } } diff --git a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/configuration/batch/steps/DumpRSAccessionsTest.java b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/configuration/batch/steps/DumpRSAccessionsTest.java index 2d3cb276e..5308c0b55 100644 --- a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/configuration/batch/steps/DumpRSAccessionsTest.java +++ b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/configuration/batch/steps/DumpRSAccessionsTest.java @@ -16,24 +16,19 @@ package uk.ac.ebi.eva.accession.release.configuration.batch.steps; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; -import com.mongodb.MongoClient; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.batch.core.Job; import org.springframework.batch.test.JobLauncherTestUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.context.ApplicationContext; import org.springframework.context.annotation.Bean; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.ampt2d.commons.accession.core.models.EventType; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpClusteredVariantEntity; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpClusteredVariantInactiveEntity; @@ -41,9 +36,10 @@ import uk.ac.ebi.eva.accession.core.model.eva.ClusteredVariantEntity; import uk.ac.ebi.eva.accession.core.model.eva.ClusteredVariantInactiveEntity; import uk.ac.ebi.eva.accession.core.model.eva.ClusteredVariantOperationEntity; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; +import uk.ac.ebi.eva.accession.release.test.configuration.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.accession.release.test.configuration.MongoTestConfiguration; import uk.ac.ebi.eva.accession.release.test.configuration.ReleaseFromDBTestConfiguration; -import uk.ac.ebi.eva.accession.release.test.rule.FixSpringMongoDbRule; import uk.ac.ebi.eva.commons.core.models.VariantType; import java.io.IOException; @@ -55,17 +51,17 @@ import java.util.Set; import java.util.stream.Collectors; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import static uk.ac.ebi.eva.accession.release.configuration.BeanNames.DUMP_ACTIVE_ACCESSIONS_JOB; import static uk.ac.ebi.eva.accession.release.configuration.BeanNames.DUMP_MERGED_AND_DEPRECATED_ACCESSIONS_JOB; import static uk.ac.ebi.eva.accession.release.test.configuration.ReleaseFromDBTestConfiguration.TEST_DUMP_ACTIVE_ACCESSIONS_JOB; import static uk.ac.ebi.eva.accession.release.test.configuration.ReleaseFromDBTestConfiguration.TEST_DUMP_MERGED_AND_DEPRECATED_ACCESSIONS_JOB; -@RunWith(SpringRunner.class) -@ContextConfiguration(classes = {MongoTestConfiguration.class, ReleaseFromDBTestConfiguration.class}) +@ExtendWith(SpringExtension.class) +@ContextConfiguration(classes = {MongoTestConfiguration.class, ReleaseFromDBTestConfiguration.class, + BatchJobRepositoryTestConfiguration.class}) @TestPropertySource("classpath:dump-rs-accession-test.properties") -public class DumpRSAccessionsTest { - private static final String TEST_DB = "test-db"; +public class DumpRSAccessionsTest extends MongoTestContainerHelper { private static final String accDumpFile = "src/test/resources/accDumpFile.csv"; @Autowired @@ -76,29 +72,18 @@ public class DumpRSAccessionsTest { @Qualifier(TEST_DUMP_MERGED_AND_DEPRECATED_ACCESSIONS_JOB) private JobLauncherTestUtils jobLauncherDumpMergedAndDeprecatedAccessions; - @Autowired - private MongoClient mongoClient; - @Autowired private MongoTemplate mongoTemplate; - //Required by nosql-unit - @Autowired - private ApplicationContext applicationContext; - - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName(TEST_DB).build()); - - @Before + @BeforeEach public void setUp() throws IOException { - this.mongoClient.dropDatabase(TEST_DB); + mongoTemplate.getDb().drop(); Files.deleteIfExists(Paths.get(accDumpFile)); } - @After + @AfterEach public void tearDown() throws IOException { - this.mongoClient.dropDatabase(TEST_DB); + mongoTemplate.getDb().drop(); Files.deleteIfExists(Paths.get(accDumpFile)); } diff --git a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/configuration/batch/steps/RSAccessionsReleaseFromDBTest.java b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/configuration/batch/steps/RSAccessionsReleaseFromDBTest.java index 4ae5dd4a3..527f11039 100644 --- a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/configuration/batch/steps/RSAccessionsReleaseFromDBTest.java +++ b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/configuration/batch/steps/RSAccessionsReleaseFromDBTest.java @@ -16,24 +16,19 @@ package uk.ac.ebi.eva.accession.release.configuration.batch.steps; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; -import com.mongodb.MongoClient; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.batch.core.BatchStatus; import org.springframework.batch.core.JobExecution; import org.springframework.batch.test.JobLauncherTestUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.context.ApplicationContext; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.ampt2d.commons.accession.core.models.EventType; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpClusteredVariantEntity; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpSubmittedVariantEntity; @@ -43,11 +38,12 @@ import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantInactiveEntity; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantOperationEntity; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.release.parameters.InputParameters; import uk.ac.ebi.eva.accession.release.parameters.ReportPathResolver; +import uk.ac.ebi.eva.accession.release.test.configuration.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.accession.release.test.configuration.MongoTestConfiguration; import uk.ac.ebi.eva.accession.release.test.configuration.ReleaseFromDBTestConfiguration; -import uk.ac.ebi.eva.accession.release.test.rule.FixSpringMongoDbRule; import uk.ac.ebi.eva.commons.core.models.VariantType; import uk.ac.ebi.eva.commons.core.utils.FileUtils; @@ -70,15 +66,15 @@ import java.util.Set; import java.util.stream.Collectors; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import static uk.ac.ebi.eva.accession.release.test.configuration.ReleaseFromDBTestConfiguration.TEST_RELEASE_ACTIVE_ACCESSIONS_JOB; import static uk.ac.ebi.eva.accession.release.test.configuration.ReleaseFromDBTestConfiguration.TEST_RELEASE_MERGED_AND_DEPRECATED_ACCESSIONS_JOB; -@RunWith(SpringRunner.class) -@ContextConfiguration(classes = {MongoTestConfiguration.class, ReleaseFromDBTestConfiguration.class}) +@ExtendWith(SpringExtension.class) +@ContextConfiguration(classes = {MongoTestConfiguration.class, ReleaseFromDBTestConfiguration.class, + BatchJobRepositoryTestConfiguration.class}) @TestPropertySource("classpath:dump-rs-accession-test.properties") -public class RSAccessionsReleaseFromDBTest { - private static final String TEST_DB = "test-db"; +public class RSAccessionsReleaseFromDBTest extends MongoTestContainerHelper { private static final String testRunDir = "src/test/resources/release-test-run"; private static final String rsAccFile = "src/test/resources/release-test-run/rsAccFile.csv"; @@ -93,32 +89,21 @@ public class RSAccessionsReleaseFromDBTest { @Qualifier(TEST_RELEASE_MERGED_AND_DEPRECATED_ACCESSIONS_JOB) private JobLauncherTestUtils jobLauncherReleaseMergedAndDeprecatedAccessions; - @Autowired - private MongoClient mongoClient; - @Autowired private MongoTemplate mongoTemplate; - //Required by nosql-unit - @Autowired - private ApplicationContext applicationContext; - - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName(TEST_DB).build()); - - @Before + @BeforeEach public void setUp() throws IOException { - this.mongoClient.dropDatabase(TEST_DB); + mongoTemplate.getDb().drop(); if (Files.exists(Paths.get(testRunDir))) { deleteDirectory(Paths.get(testRunDir)); } Files.createDirectories(Paths.get(testRunDir)); } - @After + @AfterEach public void tearDown() throws IOException { - this.mongoClient.dropDatabase(TEST_DB); + mongoTemplate.getDb().drop(); if (Files.exists(Paths.get(testRunDir))) { deleteDirectory(Paths.get(testRunDir)); } diff --git a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/test/configuration/BatchJobRepositoryTestConfiguration.java b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/test/configuration/BatchJobRepositoryTestConfiguration.java new file mode 100644 index 000000000..a4ef0782d --- /dev/null +++ b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/test/configuration/BatchJobRepositoryTestConfiguration.java @@ -0,0 +1,22 @@ +package uk.ac.ebi.eva.accession.release.test.configuration; + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Primary; +import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseBuilder; +import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseType; + +import javax.sql.DataSource; + +@Configuration +public class BatchJobRepositoryTestConfiguration { + + @Bean + @Primary + public DataSource dataSource() { + return new EmbeddedDatabaseBuilder() + .setType(EmbeddedDatabaseType.HSQL) + .addScript("org/springframework/batch/core/schema-hsqldb.sql") + .build(); + } +} \ No newline at end of file diff --git a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/test/configuration/MongoTestConfiguration.java b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/test/configuration/MongoTestConfiguration.java index 9c5fff6ca..3051afd22 100644 --- a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/test/configuration/MongoTestConfiguration.java +++ b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/test/configuration/MongoTestConfiguration.java @@ -17,12 +17,30 @@ */ package uk.ac.ebi.eva.accession.release.test.configuration; +import com.mongodb.ReadPreference; +import com.mongodb.WriteConcern; +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoClients; +import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.domain.EntityScan; import org.springframework.boot.test.autoconfigure.data.mongo.AutoConfigureDataMongo; +import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Primary; +import org.springframework.data.mongodb.MongoDatabaseFactory; import org.springframework.data.mongodb.config.EnableMongoAuditing; +import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.data.mongodb.core.SimpleMongoClientDatabaseFactory; +import org.springframework.data.mongodb.core.WriteResultChecking; +import org.springframework.data.mongodb.core.convert.DefaultDbRefResolver; +import org.springframework.data.mongodb.core.convert.DefaultMongoTypeMapper; +import org.springframework.data.mongodb.core.convert.MappingMongoConverter; +import org.springframework.data.mongodb.core.convert.MongoCustomConversions; +import org.springframework.data.mongodb.core.mapping.MongoMappingContext; import org.springframework.data.mongodb.repository.config.EnableMongoRepositories; +import java.util.Collections; + @Configuration @EntityScan(basePackages = {"uk.ac.ebi.eva.accession.core.persistence"}) @EnableMongoRepositories(basePackages = { @@ -32,4 +50,56 @@ @AutoConfigureDataMongo public class MongoTestConfiguration { + @Value("${spring.data.mongodb.uri}") + private String mongoUri; + + @Value("${spring.data.mongodb.database}") + private String database; + + @Value("${mongodb.read-preference:primary}") + private String readPreference; + + @Bean + @Primary + public MongoClient mongoClient() { + return MongoClients.create(mongoUri); + } + + @Bean("primaryFactory") + @Primary + public MongoDatabaseFactory mongoDbFactory() { + return new SimpleMongoClientDatabaseFactory(mongoClient(), database); + } + + @Bean + @Primary + public MongoCustomConversions mongoCustomConversions() { + return new MongoCustomConversions(Collections.emptyList()); + } + + @Bean + @Primary + public MongoMappingContext mongoMappingContext(MongoCustomConversions conversions) { + MongoMappingContext context = new MongoMappingContext(); + context.setSimpleTypeHolder(conversions.getSimpleTypeHolder()); + context.setAutoIndexCreation(false); + return context; + } + + @Bean + @Primary + public MongoTemplate mongoTemplate(MongoMappingContext mongoMappingContext, + MongoCustomConversions conversions) { + MappingMongoConverter converter = new MappingMongoConverter( + new DefaultDbRefResolver(mongoDbFactory()), mongoMappingContext); + converter.setCustomConversions(conversions); + converter.setTypeMapper(new DefaultMongoTypeMapper(null)); + converter.afterPropertiesSet(); + MongoTemplate mongoTemplate = new MongoTemplate(mongoDbFactory(), converter); + mongoTemplate.setReadPreference(ReadPreference.valueOf(readPreference)); + mongoTemplate.setWriteConcern(WriteConcern.MAJORITY); + mongoTemplate.setWriteResultChecking(WriteResultChecking.EXCEPTION); + return mongoTemplate; + } + } diff --git a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/test/configuration/ReleaseFromDBTestConfiguration.java b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/test/configuration/ReleaseFromDBTestConfiguration.java index 4ee537858..8f84a0cf6 100644 --- a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/test/configuration/ReleaseFromDBTestConfiguration.java +++ b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/test/configuration/ReleaseFromDBTestConfiguration.java @@ -1,8 +1,9 @@ package uk.ac.ebi.eva.accession.release.test.configuration; import org.springframework.batch.core.Job; +import org.springframework.batch.core.launch.JobLauncher; +import org.springframework.batch.core.repository.JobRepository; import org.springframework.batch.test.JobLauncherTestUtils; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.context.annotation.Bean; @@ -57,46 +58,45 @@ public JobExecutionApplicationListener jobExecutionApplicationListener() { } @Bean(TEST_DUMP_ACTIVE_ACCESSIONS_JOB) - public JobLauncherTestUtils jobLauncherTestUtilsActiveAccessions() { - return new JobLauncherTestUtils() { - @Override - @Autowired - public void setJob(@Qualifier(DUMP_ACTIVE_ACCESSIONS_JOB) Job job) { - super.setJob(job); - } - }; + public JobLauncherTestUtils jobLauncherTestUtilsActiveAccessions(JobLauncher jobLauncher, JobRepository jobRepository, + @Qualifier(DUMP_ACTIVE_ACCESSIONS_JOB) Job job) { + JobLauncherTestUtils utils = new JobLauncherTestUtils(); + utils.setJobLauncher(jobLauncher); + utils.setJobRepository(jobRepository); + utils.setJob(job); + return utils; } @Bean(TEST_DUMP_MERGED_AND_DEPRECATED_ACCESSIONS_JOB) - public JobLauncherTestUtils jobLauncherTestUtilsMergedAccessions() { - return new JobLauncherTestUtils() { - @Override - @Autowired - public void setJob(@Qualifier(DUMP_MERGED_AND_DEPRECATED_ACCESSIONS_JOB) Job job) { - super.setJob(job); - } - }; + public JobLauncherTestUtils jobLauncherTestUtilsMergedAccessions(JobLauncher jobLauncher, JobRepository jobRepository, + @Qualifier(DUMP_MERGED_AND_DEPRECATED_ACCESSIONS_JOB) Job job) { + JobLauncherTestUtils utils = new JobLauncherTestUtils(); + utils.setJobLauncher(jobLauncher); + utils.setJobRepository(jobRepository); + utils.setJob(job); + return utils; + } @Bean(TEST_RELEASE_ACTIVE_ACCESSIONS_JOB) - public JobLauncherTestUtils jobLauncherTestUtilsReleaseActiveAccessions() { - return new JobLauncherTestUtils() { - @Override - @Autowired - public void setJob(@Qualifier(ACTIVE_ACCESSIONS_RELEASE_FROM_DB_JOB) Job job) { - super.setJob(job); - } - }; + public JobLauncherTestUtils jobLauncherTestUtilsReleaseActiveAccessions(JobLauncher jobLauncher, JobRepository jobRepository, + @Qualifier(ACTIVE_ACCESSIONS_RELEASE_FROM_DB_JOB) Job job) { + JobLauncherTestUtils utils = new JobLauncherTestUtils(); + utils.setJobLauncher(jobLauncher); + utils.setJobRepository(jobRepository); + utils.setJob(job); + return utils; + } @Bean(TEST_RELEASE_MERGED_AND_DEPRECATED_ACCESSIONS_JOB) - public JobLauncherTestUtils jobLauncherTestUtilsReleaseMergedAccessions() { - return new JobLauncherTestUtils() { - @Override - @Autowired - public void setJob(@Qualifier(MERGED_AND_DEPRECATED_ACCESSIONS_RELEASE_FROM_DB_JOB) Job job) { - super.setJob(job); - } - }; + public JobLauncherTestUtils jobLauncherTestUtilsReleaseMergedAccessions(JobLauncher jobLauncher, JobRepository jobRepository, + @Qualifier(MERGED_AND_DEPRECATED_ACCESSIONS_RELEASE_FROM_DB_JOB) Job job) { + JobLauncherTestUtils utils = new JobLauncherTestUtils(); + utils.setJobLauncher(jobLauncher); + utils.setJobRepository(jobRepository); + utils.setJob(job); + return utils; + } } diff --git a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/test/rule/FixSpringMongoDbRule.java b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/test/rule/FixSpringMongoDbRule.java deleted file mode 100644 index 144884962..000000000 --- a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/test/rule/FixSpringMongoDbRule.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * - * Copyright 2018 EMBL - European Bioinformatics Institute - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -package uk.ac.ebi.eva.accession.release.test.rule; - -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfiguration; -import com.lordofthejars.nosqlunit.mongodb.SpringMongoDbRule; - -/** - * Temporary fix until nosql unit rc-6 or final is released - */ -public class FixSpringMongoDbRule extends SpringMongoDbRule { - - public FixSpringMongoDbRule(MongoDbConfiguration mongoDbConfiguration) { - super(mongoDbConfiguration); - } - - public FixSpringMongoDbRule(MongoDbConfiguration mongoDbConfiguration, Object object) { - super(mongoDbConfiguration, object); - } - - @Override - public void close() { - // DO NOT CLOSE the connection (Spring will do it when destroying the context) - } - -} diff --git a/eva-accession-release/src/test/resources/application.properties b/eva-accession-release/src/test/resources/application.properties index 2f8beb508..b8fb0bd6e 100644 --- a/eva-accession-release/src/test/resources/application.properties +++ b/eva-accession-release/src/test/resources/application.properties @@ -18,9 +18,6 @@ parameters.contigNaming=SEQUENCE_NAME spring.data.mongodb.database=test-db -spring.data.mongodb.host=|eva.mongo.host.test| -spring.data.mongodb.password= -spring.data.mongodb.port=27017 mongodb.read-preference=primary # See https://github.com/spring-projects/spring-boot/wiki/Spring-Boot-2.1-Release-Notes#bean-overriding diff --git a/eva-accession-release/src/test/resources/dump-rs-accession-test.properties b/eva-accession-release/src/test/resources/dump-rs-accession-test.properties index 2cdd166f9..ee7516341 100644 --- a/eva-accession-release/src/test/resources/dump-rs-accession-test.properties +++ b/eva-accession-release/src/test/resources/dump-rs-accession-test.properties @@ -21,9 +21,6 @@ parameters.rsAccFile=src/test/resources/release-test-run/rsAccFile.csv spring.data.mongodb.database=test-db -spring.data.mongodb.host=|eva.mongo.host.test| -spring.data.mongodb.password= -spring.data.mongodb.port=27017 mongodb.read-preference=primary # See https://github.com/spring-projects/spring-boot/wiki/Spring-Boot-2.1-Release-Notes#bean-overriding diff --git a/eva-accession-release/src/test/resources/incremental-release-pipeline-test.properties b/eva-accession-release/src/test/resources/incremental-release-pipeline-test.properties index bc8a54d38..70140cc36 100644 --- a/eva-accession-release/src/test/resources/incremental-release-pipeline-test.properties +++ b/eva-accession-release/src/test/resources/incremental-release-pipeline-test.properties @@ -17,9 +17,6 @@ parameters.chunkSize=5 parameters.contigNaming=INSDC spring.data.mongodb.database=test-db -spring.data.mongodb.host=|eva.mongo.host.test| -spring.data.mongodb.password= -spring.data.mongodb.port=27017 mongodb.read-preference=primary # See https://github.com/spring-projects/spring-boot/wiki/Spring-Boot-2.1-Release-Notes#bean-overriding diff --git a/eva-accession-release/src/test/resources/release-pipeline-test.properties b/eva-accession-release/src/test/resources/release-pipeline-test.properties index de388d415..48df3dc06 100644 --- a/eva-accession-release/src/test/resources/release-pipeline-test.properties +++ b/eva-accession-release/src/test/resources/release-pipeline-test.properties @@ -18,9 +18,6 @@ parameters.contigNaming=SEQUENCE_NAME spring.data.mongodb.database=test-db -spring.data.mongodb.host=|eva.mongo.host.test| -spring.data.mongodb.password= -spring.data.mongodb.port=27017 mongodb.read-preference=primary # See https://github.com/spring-projects/spring-boot/wiki/Spring-Boot-2.1-Release-Notes#bean-overriding diff --git a/eva-accession-ws/pom.xml b/eva-accession-ws/pom.xml index 128a7f898..a9e697f1c 100644 --- a/eva-accession-ws/pom.xml +++ b/eva-accession-ws/pom.xml @@ -3,10 +3,11 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 + uk.ac.ebi.eva eva-accession - 0.6.57-SNAPSHOT + ${revision} eva-accession-ws @@ -37,20 +38,16 @@ - com.fasterxml.jackson.datatype - jackson-datatype-jsr310 - - - com.fasterxml.jackson.core - jackson-databind - - - io.springfox - springfox-swagger2 + org.springdoc + springdoc-openapi-starter-webmvc-ui + - io.springfox - springfox-swagger-ui + uk.ac.ebi.eva + eva-accession-core + ${revision} + test-jar + test @@ -59,8 +56,18 @@ - com.lordofthejars - nosqlunit-mongodb + org.springframework.boot + spring-boot-testcontainers + test + + + org.testcontainers + mongodb + test + + + org.testcontainers + junit-jupiter test @@ -98,7 +105,6 @@ - localhost ${maven.build.timestamp} yyyyMMddHHmm diff --git a/eva-accession-ws/src/main/java/uk/ac/ebi/eva/accession/ws/EvaAccessionApplication.java b/eva-accession-ws/src/main/java/uk/ac/ebi/eva/accession/ws/EvaAccessionApplication.java index becc9a0e4..a4f3e3c30 100644 --- a/eva-accession-ws/src/main/java/uk/ac/ebi/eva/accession/ws/EvaAccessionApplication.java +++ b/eva-accession-ws/src/main/java/uk/ac/ebi/eva/accession/ws/EvaAccessionApplication.java @@ -26,7 +26,7 @@ @SpringBootApplication(exclude = {MongoAutoConfiguration.class, MongoDataAutoConfiguration.class}) public class EvaAccessionApplication extends SpringBootServletInitializer { - public static void main(String[] args) throws Exception { + public static void main(String[] args) { SpringApplication.run(EvaAccessionApplication.class, args); } } diff --git a/eva-accession-ws/src/main/java/uk/ac/ebi/eva/accession/ws/configuration/ApplicationConfiguration.java b/eva-accession-ws/src/main/java/uk/ac/ebi/eva/accession/ws/configuration/ApplicationConfiguration.java index 236559315..5a25f5e8b 100644 --- a/eva-accession-ws/src/main/java/uk/ac/ebi/eva/accession/ws/configuration/ApplicationConfiguration.java +++ b/eva-accession-ws/src/main/java/uk/ac/ebi/eva/accession/ws/configuration/ApplicationConfiguration.java @@ -21,38 +21,30 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializationFeature; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; -import org.springframework.beans.factory.ObjectProvider; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.autoconfigure.AutoConfigureAfter; -import org.springframework.boot.autoconfigure.http.HttpMessageConverters; import org.springframework.boot.autoconfigure.http.HttpMessageConvertersAutoConfiguration; -import org.springframework.boot.autoconfigure.web.client.RestTemplateAutoConfiguration; +import org.springframework.boot.autoconfigure.web.client.RestTemplateBuilderConfigurer; import org.springframework.boot.autoconfigure.web.reactive.function.client.WebClientAutoConfiguration; import org.springframework.boot.web.client.RestTemplateBuilder; -import org.springframework.boot.web.client.RestTemplateCustomizer; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Import; -import org.springframework.http.client.ClientHttpRequestFactory; import org.springframework.web.servlet.config.annotation.CorsRegistry; import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; -import org.springframework.web.servlet.config.annotation.WebMvcConfigurerAdapter; import uk.ac.ebi.ampt2d.commons.accession.autoconfigure.EnableBasicRestControllerAdvice; import uk.ac.ebi.ampt2d.commons.accession.rest.controllers.BasicRestController; - +import uk.ac.ebi.eva.accession.core.configuration.human.HumanClusteredVariantAccessioningConfiguration; +import uk.ac.ebi.eva.accession.core.configuration.nonhuman.ClusteredVariantAccessioningConfiguration; +import uk.ac.ebi.eva.accession.core.configuration.nonhuman.SubmittedVariantAccessioningConfiguration; import uk.ac.ebi.eva.accession.core.model.ClusteredVariant; -import uk.ac.ebi.eva.accession.core.service.nonhuman.ClusteredVariantAccessioningService; import uk.ac.ebi.eva.accession.core.model.IClusteredVariant; import uk.ac.ebi.eva.accession.core.model.ISubmittedVariant; import uk.ac.ebi.eva.accession.core.model.SubmittedVariant; +import uk.ac.ebi.eva.accession.core.service.nonhuman.ClusteredVariantAccessioningService; import uk.ac.ebi.eva.accession.core.service.nonhuman.SubmittedVariantAccessioningService; -import uk.ac.ebi.eva.accession.core.configuration.human.HumanClusteredVariantAccessioningConfiguration; -import uk.ac.ebi.eva.accession.core.configuration.nonhuman.ClusteredVariantAccessioningConfiguration; -import uk.ac.ebi.eva.accession.core.configuration.nonhuman.SubmittedVariantAccessioningConfiguration; import uk.ac.ebi.eva.accession.ws.response.NonRedirectingClientHttpRequestFactory; -import java.util.function.Supplier; - @Configuration @EnableBasicRestControllerAdvice @Import({ClusteredVariantAccessioningConfiguration.class, SubmittedVariantAccessioningConfiguration.class, @@ -83,7 +75,7 @@ public BasicRestController ba @Bean public WebMvcConfigurer corsConfigurer() { - return new WebMvcConfigurerAdapter() { + return new WebMvcConfigurer() { @Override public void addCorsMappings(CorsRegistry registry) { registry.addMapping("/**"); @@ -97,18 +89,8 @@ public void addCorsMappings(CorsRegistry registry) { * {@link NonRedirectingClientHttpRequestFactory}. */ @Bean - public RestTemplateBuilder restTemplateBuilder( - ObjectProvider messageConverters, - ObjectProvider restTemplateCustomizers) { - - RestTemplateAutoConfiguration restTemplateConfiguration = - new RestTemplateAutoConfiguration( - messageConverters, restTemplateCustomizers); - RestTemplateBuilder builder = restTemplateConfiguration.restTemplateBuilder(); - - Supplier supplier = NonRedirectingClientHttpRequestFactory::new; - builder = builder.requestFactory(supplier); - - return builder; + public RestTemplateBuilder restTemplateBuilder(RestTemplateBuilderConfigurer configurer) { + return configurer.configure(new RestTemplateBuilder()) + .requestFactory(NonRedirectingClientHttpRequestFactory::new); } } diff --git a/eva-accession-ws/src/main/java/uk/ac/ebi/eva/accession/ws/configuration/SwaggerConfiguration.java b/eva-accession-ws/src/main/java/uk/ac/ebi/eva/accession/ws/configuration/SwaggerConfiguration.java index ee804a623..f7a89f57c 100644 --- a/eva-accession-ws/src/main/java/uk/ac/ebi/eva/accession/ws/configuration/SwaggerConfiguration.java +++ b/eva-accession-ws/src/main/java/uk/ac/ebi/eva/accession/ws/configuration/SwaggerConfiguration.java @@ -16,41 +16,41 @@ package uk.ac.ebi.eva.accession.ws.configuration; +import io.swagger.v3.oas.models.OpenAPI; +import io.swagger.v3.oas.models.info.Contact; +import io.swagger.v3.oas.models.info.Info; +import io.swagger.v3.oas.models.info.License; +import org.springdoc.core.models.GroupedOpenApi; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import springfox.documentation.builders.ApiInfoBuilder; -import springfox.documentation.builders.PathSelectors; -import springfox.documentation.builders.RequestHandlerSelectors; -import springfox.documentation.service.ApiInfo; -import springfox.documentation.service.Contact; -import springfox.documentation.spi.DocumentationType; -import springfox.documentation.spring.web.plugins.Docket; -import springfox.documentation.swagger2.annotations.EnableSwagger2; @Configuration -@EnableSwagger2 public class SwaggerConfiguration { @Bean - public Docket productApi() { - return new Docket(DocumentationType.SWAGGER_2) - .apiInfo(getApiInfo()) - .select() - .apis(RequestHandlerSelectors.basePackage("uk.ac.ebi.eva.accession.ws.rest")) - .paths(PathSelectors.any()) + public GroupedOpenApi productApi() { + return GroupedOpenApi.builder() + .group("eva-accession-api") + .packagesToScan("uk.ac.ebi.eva.accession.ws.rest") + .pathsToMatch("/**") .build(); } - private ApiInfo getApiInfo() { - return new ApiInfoBuilder() - .contact(new Contact("Europe Variation Archive - EMBL-EBI", "https://www.ebi.ac.uk/eva", null)) - .license("Apache 2.0") - .licenseUrl("http://www.apache.org/licenses/LICENSE-2.0") - .title("EVA variant identifiers API") - .description( - "API to retrieve information about submitted variant identifiers (usually called SubSNP or " + - "SS) and clustered variant identifiers (usually called RefSNP or RS).") - .version("0.1") - .build(); + @Bean + public OpenAPI customOpenAPI() { + return new OpenAPI().info( + new Info() + .title("EVA variant identifiers API") + .description( + "API to retrieve information about submitted variant identifiers " + + "(usually called SubSNP or SS) and clustered variant identifiers " + + "(usually called RefSNP or RS).") + .version("0.1") + .contact(new Contact() + .name("Europe Variation Archive - EMBL-EBI") + .url("https://www.ebi.ac.uk/eva")) + .license(new License() + .name("Apache 2.0") + .url("http://www.apache.org/licenses/LICENSE-2.0"))); } } diff --git a/eva-accession-ws/src/main/java/uk/ac/ebi/eva/accession/ws/rest/ClusteredVariantsRestController.java b/eva-accession-ws/src/main/java/uk/ac/ebi/eva/accession/ws/rest/ClusteredVariantsRestController.java index 231b6ddd6..24a2a56d1 100644 --- a/eva-accession-ws/src/main/java/uk/ac/ebi/eva/accession/ws/rest/ClusteredVariantsRestController.java +++ b/eva-accession-ws/src/main/java/uk/ac/ebi/eva/accession/ws/rest/ClusteredVariantsRestController.java @@ -17,9 +17,10 @@ */ package uk.ac.ebi.eva.accession.ws.rest; -import io.swagger.annotations.Api; -import io.swagger.annotations.ApiOperation; -import io.swagger.annotations.ApiParam; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.tags.Tag; +import jakarta.servlet.http.HttpServletResponse; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Qualifier; @@ -39,7 +40,6 @@ import uk.ac.ebi.ampt2d.commons.accession.core.models.IEvent; import uk.ac.ebi.ampt2d.commons.accession.rest.dto.AccessionResponseDTO; import uk.ac.ebi.ampt2d.commons.accession.rest.dto.HistoryEventDTO; - import uk.ac.ebi.eva.accession.core.model.ClusteredVariant; import uk.ac.ebi.eva.accession.core.model.IClusteredVariant; import uk.ac.ebi.eva.accession.core.model.ISubmittedVariant; @@ -54,7 +54,6 @@ import uk.ac.ebi.eva.commons.core.models.VariantType; import uk.ac.ebi.eva.commons.core.models.contigalias.ContigNamingConvention; -import javax.servlet.http.HttpServletResponse; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -63,7 +62,7 @@ @RestController @RequestMapping(value = "/v1/clustered-variants") -@Api(tags = {"Clustered variants"}) +@Tag(name = "Clustered variants") public class ClusteredVariantsRestController { private static final Logger logger = LoggerFactory.getLogger(ClusteredVariantsRestController.class); @@ -97,14 +96,14 @@ public ClusteredVariantsRestController( * will be thrown and a redirection to an active RS will be done by {@link EvaControllerAdvice}. Although it is * not entirely correct, it was decided to return only one of those merges as redirection, doesn't matter which one. */ - @ApiOperation(value = "Find clustered variants (RS) by identifier", notes = "This endpoint returns the clustered " + @Operation(summary = "Find clustered variants (RS) by identifier", description = "This endpoint returns the clustered " + "variants (RS) represented by the given identifier. For a description of the response, see " + "https://github.com/EBIvariation/eva-accession/wiki/Import-accessions-from-dbSNP#clustered-variant-refsnp-or-rs") @GetMapping(value = "/{identifier}", produces = "application/json") public ResponseEntity>> get( - @PathVariable @ApiParam(value = "Numerical identifier of a clustered variant, e.g.: 3000000000", + @PathVariable @Parameter(description = "Numerical identifier of a clustered variant, e.g.: 3000000000", required = true) Long identifier, - @RequestParam(required = false) @ApiParam(value = "Contig naming convention desired, default is INSDC") + @RequestParam(required = false) @Parameter(description = "Contig naming convention desired, default is INSDC") ContigNamingConvention contigNamingConvention) throws AccessionMergedException, AccessionDoesNotExistException { try { @@ -112,7 +111,7 @@ public ResponseEntity(); clusteredVariants.addAll(getNonHumanClusteredVariants(identifier, contigNamingConvention)); clusteredVariants.addAll(humanService.getAllByAccession(identifier, contigNamingConvention).stream().map(this::toDTO) - .collect(Collectors.toList())); + .collect(Collectors.toList())); if (clusteredVariants.isEmpty()) { throw new AccessionDoesNotExistException(identifier); @@ -127,32 +126,32 @@ public ResponseEntity> getVariantHistory( - @PathVariable @ApiParam(value = "Numerical identifier of a clustered variant, e.g.: 43678406", + @PathVariable @Parameter(description = "Numerical identifier of a clustered variant, e.g.: 43678406", required = true) Long identifier, - @RequestParam(required = false) @ApiParam(value = "Contig naming convention desired, default is INSDC") - ContigNamingConvention contigNamingConvention) throws AccessionDoesNotExistException { + @RequestParam(required = false) @Parameter(description = "Contig naming convention desired, default is INSDC") + ContigNamingConvention contigNamingConvention) throws AccessionDoesNotExistException { List> allVariants = new ArrayList<>(); try { allVariants.addAll(getNonHumanClusteredVariants(identifier, contigNamingConvention)); allVariants.addAll(humanService.getAllByAccession(identifier, contigNamingConvention).stream().map(this::toDTO) - .collect(Collectors.toList())); + .collect(Collectors.toList())); } catch (NoSuchElementException e) { throw new ResponseStatusException(HttpStatus.NOT_FOUND, e.getMessage()); } catch (AccessionDeprecatedException e) { allVariants.addAll(getDeprecatedClusteredVariant(identifier)); - }catch (AccessionMergedException e){ + } catch (AccessionMergedException e) { // if accession has been merged into several other accessions, we are not getting those accessions // we will rely on operations to provide info about those rs ids in which the given rs has been merged } List> allOperations = clusteredVariantOperationService.getAllOperations(identifier, contigNamingConvention) - .stream().map(this::toHistoryEventDTO).collect(Collectors.toList()); + .stream().map(this::toHistoryEventDTO).collect(Collectors.toList()); if (allVariants.isEmpty() && allOperations.isEmpty()) { throw new AccessionDoesNotExistException(identifier); @@ -173,7 +172,7 @@ private List> getSubmittedVariants( - @PathVariable @ApiParam(value = "Numerical identifier of a clustered variant, e.g.: 869808637", + @PathVariable @Parameter(description = "Numerical identifier of a clustered variant, e.g.: 869808637", required = true) Long identifier, - @RequestParam(required = false) @ApiParam(value = "Contig naming convention desired, default is INSDC") - ContigNamingConvention contigNamingConvention) + @RequestParam(required = false) @Parameter(description = "Contig naming convention desired, default is INSDC") + ContigNamingConvention contigNamingConvention) throws AccessionDoesNotExistException, AccessionDeprecatedException, AccessionMergedException { try { // trigger the checks. if the identifier was merged, the EvaControllerAdvice will redirect to the correct @@ -209,37 +208,37 @@ public List> submittedVariants = submittedVariantsService.getByClusteredVariantAccessionIn(Collections.singletonList(identifier), - contigNamingConvention); + contigNamingConvention); return submittedVariants.stream() - .map(wrapper -> new AccessionResponseDTO<>(wrapper, SubmittedVariant::new)) - .collect(Collectors.toList()); + .map(wrapper -> new AccessionResponseDTO<>(wrapper, SubmittedVariant::new)) + .collect(Collectors.toList()); } catch (NoSuchElementException e) { throw new ResponseStatusException(HttpStatus.NOT_FOUND, e.getMessage()); } } - @ApiOperation(value = "Find a clustered variant (RS) by the identifying fields", notes = "This endpoint returns " + @Operation(summary = "Find a clustered variant (RS) by the identifying fields", description = "This endpoint returns " + "the clustered variant (RS) represented by a given identifier. For a description of the response, see " + "https://github.com/EBIvariation/eva-accession/wiki/Import-accessions-from-dbSNP#clustered-variant-refsnp" + "-or-rs") @GetMapping(produces = "application/json") public ResponseEntity>> getByIdFields( - @RequestParam(name = "assemblyId") @ApiParam(value = "assembly accession in GCA format, e.g.: GCA_000002305.1") - String assembly, - @RequestParam(name = "referenceName") @ApiParam(value = "chromosome name or accession, e.g.: CM000392.2") - String chromosome, - @RequestParam(name = "start") @ApiParam(value = "start position, e.g.: 66275332") long start, + @RequestParam(name = "assemblyId") @Parameter(description = "assembly accession in GCA format, e.g.: GCA_000002305.1") + String assembly, + @RequestParam(name = "referenceName") @Parameter(description = "chromosome name or accession, e.g.: CM000392.2") + String chromosome, + @RequestParam(name = "start") @Parameter(description = "start position, e.g.: 66275332") long start, @RequestParam(name = "variantType") VariantType variantType, - @RequestParam(required = false) @ApiParam(value = "Chromosome naming convention used, default is INSDC") - ContigNamingConvention contigNamingConvention) { + @RequestParam(required = false) @Parameter(description = "Chromosome naming convention used, default is INSDC") + ContigNamingConvention contigNamingConvention) { try { List> clusteredVariants = new ArrayList<>(); List> nonHumanClusteredVariants = nonHumanActiveService.getByIdFields(assembly, chromosome, start, variantType, - contigNamingConvention); + contigNamingConvention); nonHumanClusteredVariants.stream().map(this::toDTO).forEach(clusteredVariants::add); List> humanClusteredVariants = @@ -258,24 +257,24 @@ private AccessionResponseDTO return new AccessionResponseDTO<>(clusteredVariantWrapper, ClusteredVariant::new); } - @ApiOperation(value = "Find if a clustered variant (RS) with the given identifying fields exists in our database", - notes = "This endpoint returns true or false to indicate if the RS ID is present. Optionally return the " + + @Operation(summary = "Find if a clustered variant (RS) with the given identifying fields exists in our database", + description = "This endpoint returns true or false to indicate if the RS ID is present. Optionally return the " + "RS ID.") @GetMapping(value = "/beacon/query", produces = "application/json") public BeaconAlleleResponse doesVariantExist( - @RequestParam(name = "assemblyId") @ApiParam(value = "assembly accession in GCA format, e.g.: GCA_000002305.1") - String assembly, - @RequestParam(name = "referenceName") @ApiParam(value = "chromosome name, e.g.: 16") - String chromosome, - @RequestParam(name = "start") @ApiParam(value = "start position, e.g.: 66275332") long start, + @RequestParam(name = "assemblyId") @Parameter(description = "assembly accession in GCA format, e.g.: GCA_000002305.1") + String assembly, + @RequestParam(name = "referenceName") @Parameter(description = "chromosome name, e.g.: 16") + String chromosome, + @RequestParam(name = "start") @Parameter(description = "start position, e.g.: 66275332") long start, @RequestParam(name = "variantType") VariantType variantType, @RequestParam(name = "includeDatasetReponses", required = false) - boolean includeDatasetReponses, + boolean includeDatasetReponses, HttpServletResponse response) { try { ContigNamingConvention contigNamingConvention = ContigNamingConvention.ENA_SEQUENCE_NAME; return beaconService.queryBeaconClusteredVariant(assembly, chromosome, start, variantType, - contigNamingConvention, includeDatasetReponses); + contigNamingConvention, includeDatasetReponses); } catch (Exception ex) { logger.error("Unexpected error in beacon query for chromosome={}, start={}, assembly={}, variantType={}", chromosome, start, assembly, variantType, ex); diff --git a/eva-accession-ws/src/main/java/uk/ac/ebi/eva/accession/ws/rest/EvaControllerAdvice.java b/eva-accession-ws/src/main/java/uk/ac/ebi/eva/accession/ws/rest/EvaControllerAdvice.java index e9ab9134b..d1c79691e 100644 --- a/eva-accession-ws/src/main/java/uk/ac/ebi/eva/accession/ws/rest/EvaControllerAdvice.java +++ b/eva-accession-ws/src/main/java/uk/ac/ebi/eva/accession/ws/rest/EvaControllerAdvice.java @@ -29,7 +29,7 @@ import uk.ac.ebi.ampt2d.commons.accession.rest.BasicRestControllerAdvice; import uk.ac.ebi.ampt2d.commons.accession.rest.dto.ErrorMessage; -import javax.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletRequest; import java.net.URI; /** diff --git a/eva-accession-ws/src/main/java/uk/ac/ebi/eva/accession/ws/rest/SubmittedVariantsRestController.java b/eva-accession-ws/src/main/java/uk/ac/ebi/eva/accession/ws/rest/SubmittedVariantsRestController.java index a320b474f..5aae8cf30 100644 --- a/eva-accession-ws/src/main/java/uk/ac/ebi/eva/accession/ws/rest/SubmittedVariantsRestController.java +++ b/eva-accession-ws/src/main/java/uk/ac/ebi/eva/accession/ws/rest/SubmittedVariantsRestController.java @@ -17,17 +17,18 @@ */ package uk.ac.ebi.eva.accession.ws.rest; -import io.swagger.annotations.Api; -import io.swagger.annotations.ApiOperation; -import io.swagger.annotations.ApiParam; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.tags.Tag; +import jakarta.servlet.http.HttpServletResponse; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PathVariable; -import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; import org.springframework.web.server.ResponseStatusException; import uk.ac.ebi.ampt2d.commons.accession.core.exceptions.AccessionDeprecatedException; @@ -35,7 +36,6 @@ import uk.ac.ebi.ampt2d.commons.accession.core.exceptions.AccessionMergedException; import uk.ac.ebi.ampt2d.commons.accession.core.models.AccessionWrapper; import uk.ac.ebi.ampt2d.commons.accession.rest.dto.AccessionResponseDTO; - import uk.ac.ebi.eva.accession.core.model.ISubmittedVariant; import uk.ac.ebi.eva.accession.core.model.SubmittedVariant; import uk.ac.ebi.eva.accession.core.service.nonhuman.SubmittedVariantAccessioningService; @@ -45,7 +45,6 @@ import uk.ac.ebi.eva.accession.ws.service.SubmittedVariantsBeaconService; import uk.ac.ebi.eva.commons.core.models.contigalias.ContigNamingConvention; -import javax.servlet.http.HttpServletResponse; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -54,7 +53,7 @@ @RestController @RequestMapping(value = "/v1/submitted-variants") -@Api(tags = {"Submitted variants"}) +@Tag(name = "Submitted variants") public class SubmittedVariantsRestController { private static final Logger logger = LoggerFactory.getLogger(SubmittedVariantsRestController.class); @@ -74,19 +73,19 @@ public SubmittedVariantsRestController( * will be thrown and a redirection to an active SS will be done by {@link EvaControllerAdvice}. Although it is * not entirely correct, it was decided to return only one of those merges as redirection, doesn't matter which one. */ - @ApiOperation(value = "Find submitted variants (SS) by identifier", notes = "This endpoint returns the submitted " + @Operation(summary = "Find submitted variants (SS) by identifier", description = "This endpoint returns the submitted " + "variants (SS) represented by a given identifier. For a description of the response, see " + "https://github.com/EBIvariation/eva-accession/wiki/Import-accessions-from-dbSNP#submitted-variant-subsnp-or-ss") @GetMapping(value = "/{identifier}", produces = "application/json") public ResponseEntity>> get( - @PathVariable @ApiParam(value = "Numerical identifier of a submitted variant, e.g.: 5000000000", - required = true) Long identifier, - @RequestParam(required = false) @ApiParam(value = "Contig naming convention desired, default is INSDC") + @PathVariable @Parameter(description = "Numerical identifier of a submitted variant, e.g.: 5000000000", + required = true) Long identifier, + @RequestParam(required = false) @Parameter(description = "Contig naming convention desired, default is INSDC") ContigNamingConvention contigNamingConvention) throws AccessionMergedException, AccessionDoesNotExistException { try { return ResponseEntity.ok(service.getAllByAccession(identifier, contigNamingConvention).stream() - .map(this::toDTO).collect(Collectors.toList())); + .map(this::toDTO).collect(Collectors.toList())); } catch (NoSuchElementException e) { throw new ResponseStatusException(HttpStatus.NOT_FOUND, e.getMessage()); } catch (AccessionDeprecatedException e) { @@ -103,7 +102,7 @@ private AccessionResponseDTO /** * Retrieve the information in the collection for inactive objects. - * + *

* This method is necessary because the behaviour of BasicRestController is to return the HttpStatus.GONE with an * error message in the body. We want instead to return the HttpStatus.GONE with the variant in the body. */ @@ -117,27 +116,26 @@ private List studies, - @RequestParam(name="start") @ApiParam(value = "start position, e.g.: 12856868") long start, - @RequestParam(name="referenceBases") @ApiParam(value = "reference base(s), e.g.: C") String reference, - @RequestParam(name="alternateBases") @ApiParam(value = "alternate base(s), e.g.: T") String alternate, + @RequestParam(name = "assemblyId") @Parameter(description = "assembly accession in GCA format, e.g.: GCA_000002305.1") + String assembly, + @RequestParam(name = "referenceName") @Parameter(description = "chromosome name, e.g.: 16") String chromosome, + @RequestParam(name = "datasetIds") @Parameter(description = "study accession, e.g.: PRJEB30116") List studies, + @RequestParam(name = "start") @Parameter(description = "start position, e.g.: 12856868") long start, + @RequestParam(name = "referenceBases") @Parameter(description = "reference base(s), e.g.: C") String reference, + @RequestParam(name = "alternateBases") @Parameter(description = "alternate base(s), e.g.: T") String alternate, HttpServletResponse response) { if (start < 1) { int responseStatus = HttpServletResponse.SC_BAD_REQUEST; response.setStatus(responseStatus); return getBeaconResponseObjectWithError(alternate, reference, chromosome, start, assembly, studies, - responseStatus, - "Please provide a positive number as start position"); + responseStatus, + "Please provide a positive number as start position"); } try { ContigNamingConvention contigNamingConvention = ContigNamingConvention.ENA_SEQUENCE_NAME; return submittedVariantsBeaconService.queryBeacon(studies, alternate, reference, chromosome, start, - assembly, contigNamingConvention, false); - } - catch (Exception ex) { + assembly, contigNamingConvention, false); + } catch (Exception ex) { logger.error("Unexpected error in beacon query for chromosome={}, start={}, assembly={}, studies={}", chromosome, start, assembly, studies, ex); int responseStatus = HttpServletResponse.SC_INTERNAL_SERVER_ERROR; @@ -152,32 +150,32 @@ private BeaconAlleleResponse getBeaconResponseObjectWithError(String alternate, int errorCode, String errorMessage) { BeaconAlleleResponse result = new BeaconAlleleResponse(); BeaconAlleleRequest request = new BeaconAlleleRequest(alternate, reference, chromosome, start, - assembly, studies, false); + assembly, studies, false); result.setAlleleRequest(request); result.setError(new BeaconError(errorCode, errorMessage)); return result; } - @ApiOperation(value = "Find submitted variants (SS) by the identifying fields", notes = "This endpoint returns " + @Operation(summary = "Find submitted variants (SS) by the identifying fields", description = "This endpoint returns " + "the submitted variants (SS) represented by a given identifier. For a description of the response, see " + "https://github.com/EBIvariation/eva-accession/wiki/Import-accessions-from-dbSNP#submitted-variant" + "-subsnp-or-ss") @GetMapping(produces = "application/json") public ResponseEntity>> getByIdFields( - @RequestParam(name="assemblyId") @ApiParam(value = "assembly accession in GCA format, e.g.: GCA_000002305.1") - String assembly, - @RequestParam(name="referenceName") @ApiParam(value = "chromosome name or accession, e.g.: CM000392.2") - String chromosome, - @RequestParam(name="datasetIds") @ApiParam(value = "study accession, e.g.: PRJEB30116") List studies, - @RequestParam(name="start") @ApiParam(value = "start position, e.g.: 12856868") long start, - @RequestParam(name="referenceBases") @ApiParam(value = "reference base(s), e.g.: C") String reference, - @RequestParam(name="alternateBases") @ApiParam(value = "alternate base(s), e.g.: T") String alternate, - @RequestParam(required = false) @ApiParam(value = "Chromosome naming convention used, default is INSDC") - ContigNamingConvention contigNamingConvention) { + @RequestParam(name = "assemblyId") @Parameter(description = "assembly accession in GCA format, e.g.: GCA_000002305.1") + String assembly, + @RequestParam(name = "referenceName") @Parameter(description = "chromosome name or accession, e.g.: CM000392.2") + String chromosome, + @RequestParam(name = "datasetIds") @Parameter(description = "study accession, e.g.: PRJEB30116") List studies, + @RequestParam(name = "start") @Parameter(description = "start position, e.g.: 12856868") long start, + @RequestParam(name = "referenceBases") @Parameter(description = "reference base(s), e.g.: C") String reference, + @RequestParam(name = "alternateBases") @Parameter(description = "alternate base(s), e.g.: T") String alternate, + @RequestParam(required = false) @Parameter(description = "Chromosome naming convention used, default is INSDC") + ContigNamingConvention contigNamingConvention) { try { return ResponseEntity.ok( submittedVariantsBeaconService.getVariantByIdFields(assembly, chromosome, studies, start, reference, - alternate, contigNamingConvention)); + alternate, contigNamingConvention)); } catch (NoSuchElementException e) { throw new ResponseStatusException(HttpStatus.NOT_FOUND, e.getMessage()); } catch (Exception e) { diff --git a/eva-accession-ws/src/main/java/uk/ac/ebi/eva/accession/ws/service/ClusteredVariantsBeaconService.java b/eva-accession-ws/src/main/java/uk/ac/ebi/eva/accession/ws/service/ClusteredVariantsBeaconService.java index 1a5982526..af73094b2 100644 --- a/eva-accession-ws/src/main/java/uk/ac/ebi/eva/accession/ws/service/ClusteredVariantsBeaconService.java +++ b/eva-accession-ws/src/main/java/uk/ac/ebi/eva/accession/ws/service/ClusteredVariantsBeaconService.java @@ -17,7 +17,6 @@ */ package uk.ac.ebi.eva.accession.ws.service; -import org.apache.commons.collections.CollectionUtils; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Service; import uk.ac.ebi.ampt2d.commons.accession.core.models.AccessionWrapper; @@ -167,7 +166,7 @@ private BeaconAlleleResponse queryBeaconClusteredVariantHuman(String referenceGe getBeaconDatasetAlleleResponsesHuman = getBeaconDatasetAlleleResponsesHuman(identifier); } return buildResponse(referenceGenome, chromosome, start, variantType, !variant.isEmpty(), - getBeaconDatasetAlleleResponsesHuman); + getBeaconDatasetAlleleResponsesHuman); } private List getBeaconDatasetAlleleResponsesHuman(String clusteredVariantAccession) { @@ -193,13 +192,13 @@ private BeaconAlleleResponse mergeResponses(BeaconAlleleResponse nonHumanRespons if (response.isExists()) { List datasetAlleleResponses = new ArrayList<>(); - if (!CollectionUtils.isEmpty(nonHumanResponse.getDatasetAlleleResponses())) { + if (nonHumanResponse.getDatasetAlleleResponses() != null && !nonHumanResponse.getDatasetAlleleResponses().isEmpty()) { datasetAlleleResponses.addAll(nonHumanResponse.getDatasetAlleleResponses()); } - if (!CollectionUtils.isEmpty(humanResponse.getDatasetAlleleResponses())) { + if (humanResponse.getDatasetAlleleResponses() != null && !humanResponse.getDatasetAlleleResponses().isEmpty()) { datasetAlleleResponses.addAll(humanResponse.getDatasetAlleleResponses()); } - if (!CollectionUtils.isEmpty(datasetAlleleResponses)) { + if (datasetAlleleResponses != null && !datasetAlleleResponses.isEmpty()) { response.setDatasetAlleleResponses(datasetAlleleResponses); } } @@ -208,8 +207,8 @@ private BeaconAlleleResponse mergeResponses(BeaconAlleleResponse nonHumanRespons } public BeaconAlleleResponse getBeaconResponseObjectWithError(String reference, long start, String assembly, - VariantType variantType, int errorCode, - String errorMessage) { + VariantType variantType, int errorCode, + String errorMessage) { BeaconAlleleRequest request = new BeaconAlleleRequest(); request.setReferenceName(Chromosome.fromValue(reference)); request.setStart(start); diff --git a/eva-accession-ws/src/test/java/uk/ac/ebi/eva/accession/ws/ClusteredVariantHistoryEndPointTest.java b/eva-accession-ws/src/test/java/uk/ac/ebi/eva/accession/ws/ClusteredVariantHistoryEndPointTest.java index e56979750..9a716ab72 100644 --- a/eva-accession-ws/src/test/java/uk/ac/ebi/eva/accession/ws/ClusteredVariantHistoryEndPointTest.java +++ b/eva-accession-ws/src/test/java/uk/ac/ebi/eva/accession/ws/ClusteredVariantHistoryEndPointTest.java @@ -17,81 +17,80 @@ */ package uk.ac.ebi.eva.accession.ws; -import com.lordofthejars.nosqlunit.annotation.UsingDataSet; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; -import com.mongodb.MongoClient; -import org.junit.After; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.web.client.TestRestTemplate; -import org.springframework.context.ApplicationContext; +import org.springframework.context.annotation.Import; import org.springframework.core.ParameterizedTypeReference; +import org.springframework.core.io.ResourceLoader; +import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.http.HttpMethod; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.ampt2d.commons.accession.core.models.EventType; import uk.ac.ebi.ampt2d.commons.accession.rest.dto.AccessionResponseDTO; import uk.ac.ebi.ampt2d.commons.accession.rest.dto.HistoryEventDTO; - import uk.ac.ebi.eva.accession.core.model.ClusteredVariant; import uk.ac.ebi.eva.accession.core.model.IClusteredVariant; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; +import uk.ac.ebi.eva.accession.core.utils.MongoTestDataLoader; import uk.ac.ebi.eva.accession.ws.dto.VariantHistory; import uk.ac.ebi.eva.accession.ws.rest.ClusteredVariantsRestController; -import uk.ac.ebi.eva.accession.ws.test.rule.FixSpringMongoDbRule; +import uk.ac.ebi.eva.accession.ws.test.MongoTestConfiguration; import uk.ac.ebi.eva.commons.core.models.contigalias.ContigNamingConvention; import java.util.Arrays; import java.util.List; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; -@RunWith(SpringRunner.class) +@ExtendWith(SpringExtension.class) @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +@Import({MongoTestConfiguration.class}) @TestPropertySource("classpath:accession-ws-test.properties") -public class ClusteredVariantHistoryEndPointTest { +public class ClusteredVariantHistoryEndPointTest extends MongoTestContainerHelper { private static final String URL = "/v1/clustered-variants/%s/history"; - public static final String TEST_DB = "eva-accession-ws-test-db"; - @Autowired private TestRestTemplate testRestTemplate; @Autowired - private MongoClient mongoClient; + private MongoTemplate mongoTemplate; @Autowired - private ClusteredVariantsRestController restController; + private ResourceLoader resourceLoader; - //Required by nosql-unit @Autowired - private ApplicationContext applicationContext; + private ClusteredVariantsRestController restController; - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName(TEST_DB).build()); + @BeforeEach + public void setUp() { + mongoTemplate.getDb().drop(); + } - @After + @AfterEach public void tearDown() { - mongoClient.dropDatabase(TEST_DB); + mongoTemplate.getDb().drop(); } private static class ClusteredVariantType extends ParameterizedTypeReference< VariantHistory> { } - @UsingDataSet(locations = {"/test-data/splitOneRsIntoMultiple.json"}) @Test @DirtiesContext public void testVariantHistorySingleRSSplitIntoMultiple1() throws Exception { + new MongoTestDataLoader(mongoTemplate, resourceLoader).loadAll("/test-data/splitOneRsIntoMultiple.json"); + long fetchHistoryOfRS = 1L; ResponseEntity> response = restController.getVariantHistory(fetchHistoryOfRS, ContigNamingConvention.INSDC); @@ -105,18 +104,19 @@ public void testVariantHistorySingleRSSplitIntoMultiple1() throws Exception { assertTrue(allVariants.stream().allMatch(v -> v.getAccession().equals(1L))); //Variant with RS 1 present in both ASM1 and ASM2 assertTrue(allVariants.stream().allMatch(v -> Arrays.asList("ASM1", "ASM2") - .contains(v.getData().getAssemblyAccession()))); + .contains(v.getData().getAssemblyAccession()))); assertTrue(allOperations.stream().allMatch(o -> o.getType().equals(EventType.RS_SPLIT))); // Every split operation from RS 1 assertTrue(allOperations.stream().allMatch(o -> o.getAccession().equals(1L))); assertTrue(allOperations.stream().allMatch(o -> Arrays.asList(3000000000L, 3000000001L, 3000000002L) - .contains(o.getSplitInto()))); + .contains(o.getSplitInto()))); } - @UsingDataSet(locations = {"/test-data/rsMerge.Json"}) @Test @DirtiesContext public void testVariantHistoryRSMerge() throws Exception { + new MongoTestDataLoader(mongoTemplate, resourceLoader).loadAll("/test-data/rsMerge.Json"); + long fetchHistoryOfRS = 1L; ResponseEntity> response = restController.getVariantHistory(fetchHistoryOfRS, ContigNamingConvention.INSDC); @@ -130,17 +130,18 @@ public void testVariantHistoryRSMerge() throws Exception { assertTrue(allVariants.stream().allMatch(v -> v.getAccession().equals(1L))); //Variant present in both ASM1 and ASM2 assertTrue(allVariants.stream().allMatch(v -> Arrays.asList("ASM1", "ASM2") - .contains(v.getData().getAssemblyAccession()))); + .contains(v.getData().getAssemblyAccession()))); assertTrue(allOperations.stream().allMatch(o -> o.getType().equals(EventType.MERGED))); // Operation RS 2 merged into 1 assertEquals(2L, allOperations.get(0).getAccession().longValue()); assertEquals(1L, allOperations.get(0).getMergedInto().longValue()); } - @UsingDataSet(locations = {"/test-data/subsequentRsSplit.json"}) @Test @DirtiesContext public void testVariantHistorySubsequentRSSplit() throws Exception { + new MongoTestDataLoader(mongoTemplate, resourceLoader).loadAll("/test-data/subsequentRsSplit.json"); + long fetchHistoryOfRS = 3000000000L; ResponseEntity> response = restController.getVariantHistory(fetchHistoryOfRS, ContigNamingConvention.INSDC); @@ -154,17 +155,18 @@ public void testVariantHistorySubsequentRSSplit() throws Exception { assertTrue(allVariants.stream().allMatch(v -> v.getAccession().equals(3000000000L))); //Variant present in both ASM2 and ASM3 assertTrue(allVariants.stream().allMatch(v -> Arrays.asList("ASM2", "ASM3") - .contains(v.getData().getAssemblyAccession()))); + .contains(v.getData().getAssemblyAccession()))); assertTrue(allOperations.stream().allMatch(o -> o.getType().equals(EventType.RS_SPLIT))); assertTrue(allOperations.stream().allMatch(o -> Arrays.asList(1L, 3000000000L).contains(o.getAccession()))); assertTrue(allOperations.stream().allMatch(o -> Arrays.asList(3000000000L, 3000000001L) - .contains(o.getSplitInto()))); + .contains(o.getSplitInto()))); } - @UsingDataSet(locations = {"/test-data/subsequentRsMerge.json"}) @Test @DirtiesContext public void testVariantHistorySubsequentRSMerge() throws Exception { + new MongoTestDataLoader(mongoTemplate, resourceLoader).loadAll("/test-data/subsequentRsMerge.json"); + long fetchHistoryOfRS = 9L; ResponseEntity> response = restController.getVariantHistory(fetchHistoryOfRS, ContigNamingConvention.INSDC); @@ -183,10 +185,11 @@ public void testVariantHistorySubsequentRSMerge() throws Exception { assertTrue(allOperations.stream().allMatch(o -> Arrays.asList(9L, 8L).contains(o.getMergedInto()))); } - @UsingDataSet(locations = {"/test-data/mixOfSplitAndMerge.json"}) @Test @DirtiesContext public void testVariantHistoryMixOfSplitAndMerge() throws Exception { + new MongoTestDataLoader(mongoTemplate, resourceLoader).loadAll("/test-data/mixOfSplitAndMerge.json"); + long fetchHistoryOfRS = 3000000000L; ResponseEntity> response = restController.getVariantHistory(fetchHistoryOfRS, ContigNamingConvention.INSDC); @@ -209,10 +212,11 @@ public void testVariantHistoryMixOfSplitAndMerge() throws Exception { assertEquals(8L, allOperations.get(1).getMergedInto().longValue()); } - @UsingDataSet(locations = {"/test-data/rsSplitsEndpoint.json"}) @Test @DirtiesContext public void testVariantHistoryServiceEndPoint() { + new MongoTestDataLoader(mongoTemplate, resourceLoader).loadAll("/test-data/rsSplitsEndpoint.json"); + int fetchHistoryOfRS = 1; String getVariantsUrl = String.format(URL, fetchHistoryOfRS); @@ -231,11 +235,11 @@ public void testVariantHistoryServiceEndPoint() { assertTrue(allVariants.stream().allMatch(v -> v.getAccession().equals(1L))); //Variant with RS 1 present in both ASM1 and ASM2 assertTrue(allVariants.stream().allMatch(v -> Arrays.asList("ASM1", "ASM2") - .contains(v.getData().getAssemblyAccession()))); + .contains(v.getData().getAssemblyAccession()))); assertTrue(allOperations.stream().allMatch(o -> o.getType().equals(EventType.RS_SPLIT))); // Every split operation from RS 1 assertTrue(allOperations.stream().allMatch(o -> o.getAccession().equals(1L))); assertTrue(allOperations.stream().allMatch(o -> Arrays.asList(3000000000L, 3000000001L, 3000000002L) - .contains(o.getSplitInto()))); + .contains(o.getSplitInto()))); } } \ No newline at end of file diff --git a/eva-accession-ws/src/test/java/uk/ac/ebi/eva/accession/ws/ClusteredVariantsRestControllerTest.java b/eva-accession-ws/src/test/java/uk/ac/ebi/eva/accession/ws/ClusteredVariantsRestControllerTest.java index c3e6af8ca..3718a92b6 100644 --- a/eva-accession-ws/src/test/java/uk/ac/ebi/eva/accession/ws/ClusteredVariantsRestControllerTest.java +++ b/eva-accession-ws/src/test/java/uk/ac/ebi/eva/accession/ws/ClusteredVariantsRestControllerTest.java @@ -18,11 +18,12 @@ package uk.ac.ebi.eva.accession.ws; import com.mongodb.BasicDBObject; +import jakarta.servlet.http.HttpServletResponse; import org.bson.Document; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.Mockito; import org.springframework.beans.factory.annotation.Autowired; @@ -39,7 +40,7 @@ import org.springframework.http.ResponseEntity; import org.springframework.mock.web.MockHttpServletResponse; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import org.springframework.web.server.ResponseStatusException; import uk.ac.ebi.ampt2d.commons.accession.core.exceptions.AccessionDeprecatedException; import uk.ac.ebi.ampt2d.commons.accession.core.exceptions.AccessionDoesNotExistException; @@ -71,8 +72,10 @@ import uk.ac.ebi.eva.accession.core.service.nonhuman.dbsnp.DbsnpClusteredVariantMonotonicAccessioningService; import uk.ac.ebi.eva.accession.core.summary.ClusteredVariantSummaryFunction; import uk.ac.ebi.eva.accession.core.summary.SubmittedVariantSummaryFunction; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.ws.rest.ClusteredVariantsRestController; import uk.ac.ebi.eva.accession.ws.service.ClusteredVariantsBeaconService; +import uk.ac.ebi.eva.accession.ws.test.MongoTestConfiguration; import uk.ac.ebi.eva.accession.ws.test.NoContigTranslationArgumentMatcher; import uk.ac.ebi.eva.commons.beacon.models.BeaconAlleleRequest; import uk.ac.ebi.eva.commons.beacon.models.BeaconAlleleResponse; @@ -81,7 +84,6 @@ import uk.ac.ebi.eva.commons.core.models.VariantType; import uk.ac.ebi.eva.commons.core.models.contigalias.ContigNamingConvention; -import javax.servlet.http.HttpServletResponse; import java.time.LocalDateTime; import java.time.Month; import java.util.ArrayList; @@ -96,22 +98,23 @@ import java.util.stream.Stream; import static junit.framework.TestCase.assertTrue; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertThrows; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.argThat; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.when; -@RunWith(SpringRunner.class) +@ExtendWith(SpringExtension.class) @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) -@Import({ClusteredVariantAccessioningConfiguration.class, SubmittedVariantAccessioningConfiguration.class}) +@Import({ClusteredVariantAccessioningConfiguration.class, SubmittedVariantAccessioningConfiguration.class, + MongoTestConfiguration.class}) @TestPropertySource("classpath:accession-ws-test.properties") -public class ClusteredVariantsRestControllerTest { +public class ClusteredVariantsRestControllerTest extends MongoTestContainerHelper { private static final String URL = "/v1/clustered-variants/"; @@ -217,7 +220,7 @@ public class ClusteredVariantsRestControllerTest { @MockBean private ContigAliasService contigAliasService; - @Before + @BeforeEach public void setUp() { dbsnpRepository.deleteAll(); dbsnpSubmittedVariantRepository.deleteAll(); @@ -235,9 +238,9 @@ public void setUp() { ClusteredVariantsBeaconService mockBeaconService = Mockito.spy( new ClusteredVariantsBeaconService(clusteredService, mockHumanService, mockService)); Mockito.doThrow(new RuntimeException("Some unexpected error")).when(mockBeaconService) - .queryBeaconClusteredVariant("GCA_ERROR", "CHROM1", 123, VariantType.SNV, ContigNamingConvention.ENA_SEQUENCE_NAME, false); + .queryBeaconClusteredVariant("GCA_ERROR", "CHROM1", 123, VariantType.SNV, ContigNamingConvention.ENA_SEQUENCE_NAME, false); Mockito.doThrow(new RuntimeException("Some unexpected error")).when(mockHumanService) - .getByIdFields("GCA_ERROR", "CHROM1", 123, VariantType.SNV, ContigNamingConvention.INSDC); + .getByIdFields("GCA_ERROR", "CHROM1", 123, VariantType.SNV, ContigNamingConvention.INSDC); mockController = new ClusteredVariantsRestController(mockService, mockBeaconService, mockHumanService, clusteredService, clusteredVariantOperationService ); @@ -245,26 +248,26 @@ public void setUp() { private void setupDbSnpClusteredVariants() { ClusteredVariant variant1 = new ClusteredVariant("ASMACC01", 1101, "CHROM1", 1234, VariantType.SNV, false, - null); + null); ClusteredVariant variant2 = new ClusteredVariant("ASMACC01", 1102, "CHROM1", 1234, VariantType.MNV, true, null); ClusteredVariant variant3 = new ClusteredVariant("ASMACC01", 1102, "CHROM1", 4567, VariantType.SNV, false, - null); + null); Function function = new ClusteredVariantSummaryFunction().andThen(new SHA1HashingFunction()); clusteredVariantEntity1 = new DbsnpClusteredVariantEntity(DBSNP_CLUSTERED_VARIANT_ACCESSION_1, - function.apply(variant1), variant1); + function.apply(variant1), variant1); clusteredVariantEntity2 = new DbsnpClusteredVariantEntity(DBSNP_CLUSTERED_VARIANT_ACCESSION_2, - function.apply(variant2), variant2); + function.apply(variant2), variant2); clusteredVariantEntity3 = new DbsnpClusteredVariantEntity(DBSNP_CLUSTERED_VARIANT_ACCESSION_3, - function.apply(variant3), variant3); + function.apply(variant3), variant3); // No new dbSNP accessions can be generated, so the variants can only be stored directly using a repository // TODO When the support for new EVA accessions is implemented, this could be changed // In order to do so, replicate the structure of {@link SubmittedVariantsRestControllerTest} generatedAccessions = dbsnpRepository.saveAll(Arrays.asList(clusteredVariantEntity1, clusteredVariantEntity2, - clusteredVariantEntity3)); + clusteredVariantEntity3)); setUpContigAliasMock(); } @@ -311,12 +314,12 @@ private void setUpContigAliasMock() { private void setupDbSnpClusteredHumanVariants() { ClusteredVariant variant1 = new ClusteredVariant("GCA_000001405.27", 9606, "CM000684.2", 45565333L, - VariantType.SNV, false, - LocalDateTime.of(2000, Month.SEPTEMBER, 19, 18, 2, 0)); + VariantType.SNV, false, + LocalDateTime.of(2000, Month.SEPTEMBER, 19, 18, 2, 0)); ClusteredVariant variant2 = new ClusteredVariant("GCA_000001405.27", 9606, "CM000663.2", 1234L, - VariantType.SNV, false, - LocalDateTime.of(2000, Month.SEPTEMBER, 19, 18, 2, 0)); + VariantType.SNV, false, + LocalDateTime.of(2000, Month.SEPTEMBER, 19, 18, 2, 0)); Function function = new ClusteredVariantSummaryFunction().andThen(new SHA1HashingFunction()); @@ -353,23 +356,23 @@ private void setupDbSnpClusteredHumanOperations() { private void setupDbsnpSubmittedVariants() { // one variant has default flags, the other have no default values SubmittedVariant submittedVariant1 = new SubmittedVariant("ASMACC01", 1101, "PROJECT1", "CHROM1", 1234, "REF", - "ALT", DBSNP_CLUSTERED_VARIANT_ACCESSION_1); + "ALT", DBSNP_CLUSTERED_VARIANT_ACCESSION_1); SubmittedVariant submittedVariant2 = new SubmittedVariant("ASMACC01", 1102, "PROJECT1", "CHROM1", 2345, "REF", - "ALT", DBSNP_CLUSTERED_VARIANT_ACCESSION_2, - !ISubmittedVariant.DEFAULT_SUPPORTED_BY_EVIDENCE, - !ISubmittedVariant.DEFAULT_ASSEMBLY_MATCH, - !ISubmittedVariant.DEFAULT_ALLELES_MATCH, - !ISubmittedVariant.DEFAULT_VALIDATED, null); + "ALT", DBSNP_CLUSTERED_VARIANT_ACCESSION_2, + !ISubmittedVariant.DEFAULT_SUPPORTED_BY_EVIDENCE, + !ISubmittedVariant.DEFAULT_ASSEMBLY_MATCH, + !ISubmittedVariant.DEFAULT_ALLELES_MATCH, + !ISubmittedVariant.DEFAULT_VALIDATED, null); SubmittedVariantSummaryFunction submittedVariantSummaryFunction = new SubmittedVariantSummaryFunction(); submittedVariantEntity1 = new DbsnpSubmittedVariantEntity(DBSNP_SUBMITTED_VARIANT_ACCESSION_1, - submittedVariantSummaryFunction.apply(submittedVariant1), - submittedVariant1, VERSION_1); + submittedVariantSummaryFunction.apply(submittedVariant1), + submittedVariant1, VERSION_1); submittedVariantEntity2 = new DbsnpSubmittedVariantEntity(DBSNP_SUBMITTED_VARIANT_ACCESSION_2, - submittedVariantSummaryFunction.apply(submittedVariant2), - submittedVariant2, VERSION_1); + submittedVariantSummaryFunction.apply(submittedVariant2), + submittedVariant2, VERSION_1); dbsnpSubmittedVariantRepository.saveAll(Arrays.asList(submittedVariantEntity1, submittedVariantEntity2)); } @@ -377,28 +380,28 @@ private void setupDbsnpSubmittedVariants() { private void setupEvaSubmittedVariants() { // one variant has no default flags, while the others have the default values SubmittedVariant submittedVariant3 = new SubmittedVariant("ASMACC01", 1102, "EVAPROJECT1", "CHROM1", 1234, - "REF", "ALT", DBSNP_CLUSTERED_VARIANT_ACCESSION_2); + "REF", "ALT", DBSNP_CLUSTERED_VARIANT_ACCESSION_2); SubmittedVariant submittedVariant4 = new SubmittedVariant("ASMACC01", 1102, "EVAPROJECT1", "CHROM1", 4567, - "REF", "ALT", DBSNP_CLUSTERED_VARIANT_ACCESSION_3, - !ISubmittedVariant.DEFAULT_SUPPORTED_BY_EVIDENCE, - !ISubmittedVariant.DEFAULT_ASSEMBLY_MATCH, - !ISubmittedVariant.DEFAULT_ALLELES_MATCH, - !ISubmittedVariant.DEFAULT_VALIDATED, null); + "REF", "ALT", DBSNP_CLUSTERED_VARIANT_ACCESSION_3, + !ISubmittedVariant.DEFAULT_SUPPORTED_BY_EVIDENCE, + !ISubmittedVariant.DEFAULT_ASSEMBLY_MATCH, + !ISubmittedVariant.DEFAULT_ALLELES_MATCH, + !ISubmittedVariant.DEFAULT_VALIDATED, null); SubmittedVariantSummaryFunction submittedVariantSummaryFunction = new SubmittedVariantSummaryFunction(); evaSubmittedVariantEntity3 = new SubmittedVariantEntity(EVA_SUBMITTED_VARIANT_ACCESSION_1, - submittedVariantSummaryFunction.apply(submittedVariant3), - submittedVariant3, VERSION_1); + submittedVariantSummaryFunction.apply(submittedVariant3), + submittedVariant3, VERSION_1); evaSubmittedVariantEntity4 = new SubmittedVariantEntity(EVA_SUBMITTED_VARIANT_ACCESSION_2, - submittedVariantSummaryFunction.apply(submittedVariant4), - submittedVariant4, VERSION_2); + submittedVariantSummaryFunction.apply(submittedVariant4), + submittedVariant4, VERSION_2); submittedVariantRepository.saveAll(Arrays.asList(evaSubmittedVariantEntity3, evaSubmittedVariantEntity4)); } - @After + @AfterEach public void tearDown() { dbsnpRepository.deleteAll(); dbsnpSubmittedVariantRepository.deleteAll(); @@ -412,10 +415,10 @@ public void tearDown() { @Test public void checkIndexInactiveObjectHashedMessageOnlyInHumanDB() { assertFalse(isIndexInCollection(mongoTemplate, DBSNP_CLUSTERED_VARIANT_OPERATION_ENTITY, - INACTIVE_OBJECTS_HASHED_MESSAGE)); + INACTIVE_OBJECTS_HASHED_MESSAGE)); assertTrue(isIndexInCollection(humanMongoTemplate, DBSNP_CLUSTERED_VARIANT_OPERATION_ENTITY, - INACTIVE_OBJECTS_HASHED_MESSAGE)); + INACTIVE_OBJECTS_HASHED_MESSAGE)); } private boolean isIndexInCollection(MongoTemplate template, String collection, String indexName) { @@ -493,11 +496,11 @@ private void checkClusteredVariantsOutput( Long accession) { List> expectedVariants = Stream.of(clusteredVariantEntity1, clusteredVariantEntity2, clusteredVariantEntity3) - .filter(v -> v.getAccession().equals(accession)) - .collect(Collectors.toList()); + .filter(v -> v.getAccession().equals(accession)) + .collect(Collectors.toList()); assertVariantsAreContainedInControllerResponse(getVariantsResponse, - expectedVariants, - ClusteredVariant::new); + expectedVariants, + ClusteredVariant::new); assertClusteredVariantCreatedDateNotNull(getVariantsResponse); } @@ -515,22 +518,22 @@ private void assertVariantsAreContainedInControllerResponse( Function modelToDto) { // check the accessions returned by the service Set retrievedAccessions = getVariantsResponse.stream() - .map(AccessionResponseDTO::getAccession) - .collect(Collectors.toSet()); + .map(AccessionResponseDTO::getAccession) + .collect(Collectors.toSet()); assertTrue(expectedVariants.stream() - .map(AccessionedDocument::getAccession) - .allMatch(retrievedAccessions::contains)); + .map(AccessionedDocument::getAccession) + .allMatch(retrievedAccessions::contains)); // check the objects returned by the service Set variantsReturnedByController = getVariantsResponse.stream() - .map(AccessionResponseDTO::getData) - .collect(Collectors.toSet()); + .map(AccessionResponseDTO::getData) + .collect(Collectors.toSet()); assertTrue(expectedVariants.stream() - .map(AccessionedDocument::getModel) - .map(modelToDto) - .allMatch(variantsReturnedByController::contains)); + .map(AccessionedDocument::getModel) + .map(modelToDto) + .allMatch(variantsReturnedByController::contains)); } private void assertClusteredVariantCreatedDateNotNull( @@ -565,13 +568,13 @@ private void checkSubmittedVariantsOutput( Long accession) { List> expectedVariants = Stream.of(submittedVariantEntity1, submittedVariantEntity2, evaSubmittedVariantEntity3, - evaSubmittedVariantEntity4) - .filter(v -> v.getAccession().equals(accession)) - .collect(Collectors.toList()); + evaSubmittedVariantEntity4) + .filter(v -> v.getAccession().equals(accession)) + .collect(Collectors.toList()); assertVariantsAreContainedInControllerResponse(getSubmittedVariantsReponse, - expectedVariants, - SubmittedVariant::new); + expectedVariants, + SubmittedVariant::new); assertSubmittedVariantCreatedDateNotNull(getSubmittedVariantsReponse); } @@ -619,12 +622,12 @@ public void testGetSubmittedVariantsByClusteredVariantIds_withContigTranslation( throws AccessionMergedException, AccessionDoesNotExistException, AccessionDeprecatedException { List> getVariantsResponse = controller.getSubmittedVariants(DBSNP_CLUSTERED_VARIANT_ACCESSION_1, - ContigNamingConvention.ENA_SEQUENCE_NAME); + ContigNamingConvention.ENA_SEQUENCE_NAME); DbsnpSubmittedVariantEntity expectedSubmittedVariant = getDbsnpSubmittedVariantEntityWithEnaContigName( submittedVariantEntity1); assertVariantsAreContainedInControllerResponse(getVariantsResponse, - Collections.singletonList(expectedSubmittedVariant), - SubmittedVariant::new); + Collections.singletonList(expectedSubmittedVariant), + SubmittedVariant::new); } private DbsnpSubmittedVariantEntity getDbsnpSubmittedVariantEntityWithEnaContigName( @@ -655,8 +658,8 @@ private void getAndCheckSubmittedVariantsByClusteredVariantIds(Long clusteredVar List> getVariantsResponse = controller.getSubmittedVariants(clusteredVariantIds, ContigNamingConvention.INSDC); assertVariantsAreContainedInControllerResponse(getVariantsResponse, - expectedSubmittedVariants, - SubmittedVariant::new); + expectedSubmittedVariants, + SubmittedVariant::new); } @Test @@ -664,13 +667,13 @@ public void testGetRedirectionForMergedVariants() throws AccessionMergedException, AccessionDoesNotExistException, AccessionDeprecatedException { // given clusteredService.merge(DBSNP_CLUSTERED_VARIANT_ACCESSION_1, - DBSNP_CLUSTERED_VARIANT_ACCESSION_2, - "Just for testing the endpoint, let's pretend the variants are equivalent"); + DBSNP_CLUSTERED_VARIANT_ACCESSION_2, + "Just for testing the endpoint, let's pretend the variants are equivalent"); // when String getVariantsUrl = URL + DBSNP_CLUSTERED_VARIANT_ACCESSION_1; ResponseEntity firstResponse = testRestTemplate.exchange(getVariantsUrl, HttpMethod.GET, null, - String.class); + String.class); // then assertEquals(HttpStatus.MOVED_PERMANENTLY, firstResponse.getStatusCode()); @@ -686,7 +689,7 @@ public void testGetRedirectionForMergedVariants() assertEquals(HttpStatus.OK, getVariantsResponse.getStatusCode()); assertEquals(1, getVariantsResponse.getBody().size()); assertEquals(new Long(DBSNP_CLUSTERED_VARIANT_ACCESSION_2), - getVariantsResponse.getBody().get(0).getAccession()); + getVariantsResponse.getBody().get(0).getAccession()); assertClusteredVariantCreatedDateNotNull(getVariantsResponse.getBody()); } @@ -695,13 +698,13 @@ public void testGetRedirectionForSubmittedVariantByMergedClusteredVariant() throws AccessionMergedException, AccessionDoesNotExistException, AccessionDeprecatedException { // given clusteredService.merge(DBSNP_CLUSTERED_VARIANT_ACCESSION_1, - DBSNP_CLUSTERED_VARIANT_ACCESSION_2, - "Just for testing the endpoint, let's pretend the variants are equivalent"); + DBSNP_CLUSTERED_VARIANT_ACCESSION_2, + "Just for testing the endpoint, let's pretend the variants are equivalent"); // when String getVariantsUrl = URL + DBSNP_CLUSTERED_VARIANT_ACCESSION_1 + "/submitted"; ResponseEntity firstResponse = testRestTemplate.exchange(getVariantsUrl, HttpMethod.GET, null, - String.class); + String.class); // then assertEquals(HttpStatus.MOVED_PERMANENTLY, firstResponse.getStatusCode()); @@ -719,7 +722,7 @@ public void testGetRedirectionForSubmittedVariantByMergedClusteredVariant() for (AccessionResponseDTO bodyEntry : getVariantsResponse.getBody()) { assertEquals(new Long(DBSNP_CLUSTERED_VARIANT_ACCESSION_2), - bodyEntry.getData().getClusteredVariantAccession()); + bodyEntry.getData().getClusteredVariantAccession()); } assertSubmittedVariantCreatedDateNotNull(getVariantsResponse.getBody()); @@ -735,46 +738,46 @@ public void testGetVariantsMergedSeveralTimes() // given Long outdatedAccession = 1L; ClusteredVariant variant1 = new ClusteredVariant("ASMACC01", 2000, "CHROM1", 1234, VariantType.SNV, false, - null); + null); DbsnpClusteredVariantEntity clusteredVariantEntity1 = new DbsnpClusteredVariantEntity(outdatedAccession, - "hash-100", variant1, 1); + "hash-100", variant1, 1); ClusteredVariant variant2 = new ClusteredVariant("ASMACC02", 2000, "CHROM2", 1234, VariantType.SNV, false, - null); + null); DbsnpClusteredVariantEntity clusteredVariantEntity2 = new DbsnpClusteredVariantEntity(outdatedAccession, - "hash-200", variant2, 1); + "hash-200", variant2, 1); Long currentAccession = 2L; ClusteredVariant variant4 = new ClusteredVariant("ASMACC03", 2000, "CHROM2", 1234, VariantType.SNV, false, - null); + null); DbsnpClusteredVariantEntity clusteredVariantEntity4 = new DbsnpClusteredVariantEntity(currentAccession, - "hash-400", variant4, 1); + "hash-400", variant4, 1); Long anotherCurrentAccession = 3L; ClusteredVariant variant5 = new ClusteredVariant("ASMACC04", 2000, "CHROM2", 1234, VariantType.SNV, false, - null); + null); DbsnpClusteredVariantEntity clusteredVariantEntity5 = new DbsnpClusteredVariantEntity(anotherCurrentAccession, - "hash-500", variant5, 1); + "hash-500", variant5, 1); mongoTemplate.dropCollection(DbsnpClusteredVariantEntity.class); mongoTemplate.insert(Arrays.asList(clusteredVariantEntity1, clusteredVariantEntity4, clusteredVariantEntity5), - DbsnpClusteredVariantEntity.class); + DbsnpClusteredVariantEntity.class); dbsnpService.merge(outdatedAccession, currentAccession, - "Just for testing the endpoint, let's pretend the variants are equivalent"); + "Just for testing the endpoint, let's pretend the variants are equivalent"); mongoTemplate.insert(Arrays.asList(clusteredVariantEntity2), DbsnpClusteredVariantEntity.class); dbsnpService.merge(outdatedAccession, anotherCurrentAccession, - "Second merge. This can totally happen importing from dbSNP. See rs106458077"); + "Second merge. This can totally happen importing from dbSNP. See rs106458077"); // when String getVariantsUrl = URL + outdatedAccession; ResponseEntity firstResponse = testRestTemplate.exchange(getVariantsUrl, HttpMethod.GET, null, - String.class); + String.class); // then assertEquals(HttpStatus.MOVED_PERMANENTLY, firstResponse.getStatusCode()); String redirectUrlIncludingHostAndPort = firstResponse.getHeaders().get(HttpHeaders.LOCATION).get(0); String redirectedUrl = redirectUrlIncludingHostAndPort.substring(redirectUrlIncludingHostAndPort.indexOf(URL)); assertTrue((URL + currentAccession).equals(redirectedUrl) - || (URL + anotherCurrentAccession).equals(redirectedUrl)); + || (URL + anotherCurrentAccession).equals(redirectedUrl)); // and then ResponseEntity>> getVariantsResponse = testRestTemplate @@ -783,7 +786,7 @@ public void testGetVariantsMergedSeveralTimes() assertEquals(HttpStatus.OK, getVariantsResponse.getStatusCode()); assertEquals(1, getVariantsResponse.getBody().size()); assertTrue(Arrays.asList(currentAccession, anotherCurrentAccession) - .contains(getVariantsResponse.getBody().get(0).getAccession())); + .contains(getVariantsResponse.getBody().get(0).getAccession())); assertClusteredVariantCreatedDateNotNull(getVariantsResponse.getBody()); } @@ -797,38 +800,38 @@ public void testGetMergedAndDeprecatedVariants() // given Long outdatedAccession = 1L; ClusteredVariant variant1 = new ClusteredVariant("ASMACC01", 2000, "CHROM1", 1234, VariantType.SNV, false, - null); + null); DbsnpClusteredVariantEntity clusteredVariantEntity1 = new DbsnpClusteredVariantEntity(outdatedAccession, - "hash-100", variant1, 1); + "hash-100", variant1, 1); ClusteredVariant variant2 = new ClusteredVariant("ASMACC02", 2000, "CHROM2", 1234, VariantType.SNV, false, - null); + null); DbsnpClusteredVariantEntity clusteredVariantEntity2 = new DbsnpClusteredVariantEntity(outdatedAccession, - "hash-200", variant2, 1); + "hash-200", variant2, 1); ClusteredVariant variant3 = new ClusteredVariant("ASMACC02", 2000, "CHROM2", 1234, VariantType.SNV, false, - null); + null); DbsnpClusteredVariantEntity clusteredVariantEntity3 = new DbsnpClusteredVariantEntity(outdatedAccession, - "hash-300", variant3, 1); + "hash-300", variant3, 1); Long currentAccession = 2L; ClusteredVariant variant4 = new ClusteredVariant("ASMACC02", 2000, "CHROM2", 1234, VariantType.SNV, false, - null); + null); DbsnpClusteredVariantEntity clusteredVariantEntity4 = new DbsnpClusteredVariantEntity(currentAccession, - "hash-400", variant4, 1); + "hash-400", variant4, 1); Long anotherCurrentAccession = 3L; ClusteredVariant variant5 = new ClusteredVariant("ASMACC01", 2000, "CHROM2", 1234, VariantType.SNV, false, - null); + null); DbsnpClusteredVariantEntity clusteredVariantEntity5 = new DbsnpClusteredVariantEntity(anotherCurrentAccession, - "hash-500", variant5, 1); + "hash-500", variant5, 1); mongoTemplate.dropCollection(DbsnpClusteredVariantEntity.class); mongoTemplate.insert(Arrays.asList(clusteredVariantEntity1, clusteredVariantEntity4, clusteredVariantEntity5), - DbsnpClusteredVariantEntity.class); + DbsnpClusteredVariantEntity.class); dbsnpService.merge(outdatedAccession, currentAccession, - "Just for testing the endpoint, let's pretend the variants are equivalent"); + "Just for testing the endpoint, let's pretend the variants are equivalent"); mongoTemplate.insert(Arrays.asList(clusteredVariantEntity2), DbsnpClusteredVariantEntity.class); dbsnpService.merge(outdatedAccession, anotherCurrentAccession, - "Second merge. This can totally happen importing from dbSNP. See rs106458077"); + "Second merge. This can totally happen importing from dbSNP. See rs106458077"); mongoTemplate.insert(Arrays.asList(clusteredVariantEntity3), DbsnpClusteredVariantEntity.class); dbsnpService.deprecate(outdatedAccession, "And then deprecated it."); @@ -899,18 +902,18 @@ public void testGetDeprecatedEvaClusteredVariant() // given Long deprecatedAccession = 1L; ClusteredVariant variant1 = new ClusteredVariant("ASMACC01", 2000, "CHROM1", 1234, VariantType.SNV, false, - null); + null); DbsnpClusteredVariantEntity clusteredVariantEntity1 = new DbsnpClusteredVariantEntity(deprecatedAccession, - "hash-100", variant1, 1); + "hash-100", variant1, 1); ClusteredVariant variant2 = new ClusteredVariant("ASMACC02", 2000, "CHROM2", 1234, VariantType.SNV, false, - null); + null); Long otherAccession = 2L; DbsnpClusteredVariantEntity clusteredVariantEntity2 = new DbsnpClusteredVariantEntity(otherAccession, - "hash-200", variant2, 1); + "hash-200", variant2, 1); mongoTemplate.dropCollection(DbsnpClusteredVariantEntity.class); mongoTemplate.insert(Arrays.asList(clusteredVariantEntity1, clusteredVariantEntity2), - DbsnpClusteredVariantEntity.class); + DbsnpClusteredVariantEntity.class); clusteredService.deprecate(deprecatedAccession, "deprecated for testing"); String getVariantUrl = URL + deprecatedAccession; @@ -929,10 +932,10 @@ public void testGetDeprecatedEvaClusteredVariant() public void testGetByIdFields() { ResponseEntity>> getVariantsResponse = controller.getByIdFields(clusteredVariantEntity1.getAssemblyAccession(), - clusteredVariantEntity1.getContig(), - clusteredVariantEntity1.getStart(), - clusteredVariantEntity1.getType(), - ContigNamingConvention.INSDC); + clusteredVariantEntity1.getContig(), + clusteredVariantEntity1.getStart(), + clusteredVariantEntity1.getType(), + ContigNamingConvention.INSDC); assertEquals(HttpStatus.OK, getVariantsResponse.getStatusCode()); assertEquals(clusteredVariantEntity1.getAccession(), getVariantsResponse.getBody().get(0).getAccession()); @@ -942,35 +945,35 @@ public void testGetByIdFields() { public void testGetByIdFields_withContigTranslation() { ResponseEntity>> getVariantsResponse = controller.getByIdFields(clusteredVariantEntity1.getAssemblyAccession(), - clusteredVariantEntity1.getContig() + ENA_CONTIG_SUFFIX, - clusteredVariantEntity1.getStart(), - clusteredVariantEntity1.getType(), - ContigNamingConvention.ENA_SEQUENCE_NAME); + clusteredVariantEntity1.getContig() + ENA_CONTIG_SUFFIX, + clusteredVariantEntity1.getStart(), + clusteredVariantEntity1.getType(), + ContigNamingConvention.ENA_SEQUENCE_NAME); assertEquals(HttpStatus.OK, getVariantsResponse.getStatusCode()); assertEquals(clusteredVariantEntity1.getAccession(), getVariantsResponse.getBody().get(0).getAccession()); assertEquals(clusteredVariantEntity1.getContig() + ENA_CONTIG_SUFFIX, - getVariantsResponse.getBody().get(0).getData().getContig()); + getVariantsResponse.getBody().get(0).getData().getContig()); } @Test public void testGetByIdFields_withWrongContigTranslation() { assertThrows(ResponseStatusException.class, - () -> controller.getByIdFields(clusteredVariantEntity1.getAssemblyAccession(), - clusteredVariantEntity1.getContig(), - clusteredVariantEntity1.getStart(), - clusteredVariantEntity1.getType(), - ContigNamingConvention.UCSC)); + () -> controller.getByIdFields(clusteredVariantEntity1.getAssemblyAccession(), + clusteredVariantEntity1.getContig(), + clusteredVariantEntity1.getStart(), + clusteredVariantEntity1.getType(), + ContigNamingConvention.UCSC)); } @Test public void testGetByIdFieldsHumanVariant() { ResponseEntity>> getVariantsResponse = controller.getByIdFields(clusteredHumanVariantEntity1.getAssemblyAccession(), - clusteredHumanVariantEntity1.getContig(), - clusteredHumanVariantEntity1.getStart(), - clusteredHumanVariantEntity1.getType(), - ContigNamingConvention.INSDC); + clusteredHumanVariantEntity1.getContig(), + clusteredHumanVariantEntity1.getStart(), + clusteredHumanVariantEntity1.getType(), + ContigNamingConvention.INSDC); assertEquals(HttpStatus.OK, getVariantsResponse.getStatusCode()); assertEquals(clusteredHumanVariantEntity1.getAccession(), getVariantsResponse.getBody().get(0).getAccession()); @@ -980,10 +983,10 @@ public void testGetByIdFieldsHumanVariant() { public void testGetByIdFieldsHumanVariantInOperations() { ResponseEntity>> getVariantsResponse = controller.getByIdFields(clusteredHumanVariantEntity3.getAssemblyAccession(), - clusteredHumanVariantEntity3.getContig(), - clusteredHumanVariantEntity3.getStart(), - clusteredHumanVariantEntity3.getType(), - ContigNamingConvention.INSDC); + clusteredHumanVariantEntity3.getContig(), + clusteredHumanVariantEntity3.getStart(), + clusteredHumanVariantEntity3.getType(), + ContigNamingConvention.INSDC); assertEquals(HttpStatus.OK, getVariantsResponse.getStatusCode()); assertEquals(clusteredHumanVariantEntity3.getAccession(), getVariantsResponse.getBody().get(0).getAccession()); @@ -993,10 +996,10 @@ public void testGetByIdFieldsHumanVariantInOperations() { public void testGetByIdFieldsHumanVariantDoesntExists() { ResponseEntity>> getVariantsResponse = controller.getByIdFields(clusteredHumanVariantEntity3.getAssemblyAccession(), - clusteredHumanVariantEntity3.getContig(), - 1L, - clusteredHumanVariantEntity3.getType(), - ContigNamingConvention.INSDC); + clusteredHumanVariantEntity3.getContig(), + 1L, + clusteredHumanVariantEntity3.getType(), + ContigNamingConvention.INSDC); assertEquals(HttpStatus.NOT_FOUND, getVariantsResponse.getStatusCode()); } @@ -1005,20 +1008,20 @@ public void testGetByIdFieldsHumanVariantDoesntExists() { public void testGetByIdFieldsClusteredVariantDoesntExists() { ResponseEntity>> getVariantsResponse = controller.getByIdFields(clusteredVariantEntity1.getAssemblyAccession(), - clusteredVariantEntity1.getContig(), - 123, - clusteredVariantEntity1.getType(), - ContigNamingConvention.INSDC); + clusteredVariantEntity1.getContig(), + 123, + clusteredVariantEntity1.getType(), + ContigNamingConvention.INSDC); assertEquals(HttpStatus.NOT_FOUND, getVariantsResponse.getStatusCode()); } - @Test(expected = RuntimeException.class) + @Test public void getByIdFieldstError500() { String assemblyId = "GCA_ERROR"; String chromosome = "CHROM1"; int start = 123; - mockController.getByIdFields(assemblyId, chromosome, start, VariantType.SNV, ContigNamingConvention.INSDC); + assertThrows(RuntimeException.class, () -> mockController.getByIdFields(assemblyId, chromosome, start, VariantType.SNV, ContigNamingConvention.INSDC)); } @Test @@ -1055,7 +1058,7 @@ private void assertDatasets(BeaconAlleleResponse beaconAlleleResponse) { private void assertEmbeddedAlleleRequest(BeaconAlleleResponse beaconAlleleResponse, DbsnpClusteredVariantEntity clusteredVariantEntity) { assertEmbeddedAlleleRequest(beaconAlleleResponse, clusteredVariantEntity.getAssemblyAccession(), - clusteredVariantEntity.getStart(), clusteredVariantEntity.getType()); + clusteredVariantEntity.getStart(), clusteredVariantEntity.getType()); } private void assertEmbeddedAlleleRequest(BeaconAlleleResponse beaconAlleleResponse, String assemblyId, @@ -1100,7 +1103,7 @@ public void doesVariantExistFalse() { assertFalse(beaconAlleleResponse.isExists()); assertNull(beaconAlleleResponse.getDatasetAlleleResponses()); assertEmbeddedAlleleRequest(beaconAlleleResponse, clusteredVariantEntity1.getAssemblyAccession(), 123L, - clusteredVariantEntity1.getType()); + clusteredVariantEntity1.getType()); } @Test @@ -1114,7 +1117,7 @@ public void doesVariantExistError500() { assertEquals(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, response.getStatus()); assertEquals(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, - (int) beaconAlleleResponse.getError().getErrorCode()); + (int) beaconAlleleResponse.getError().getErrorCode()); assertEmbeddedAlleleRequest(beaconAlleleResponse, assemblyId, start, VariantType.SNV); } } diff --git a/eva-accession-ws/src/test/java/uk/ac/ebi/eva/accession/ws/SubmittedVariantsRestControllerTest.java b/eva-accession-ws/src/test/java/uk/ac/ebi/eva/accession/ws/SubmittedVariantsRestControllerTest.java index 1b9ea0d5a..eda734d86 100644 --- a/eva-accession-ws/src/test/java/uk/ac/ebi/eva/accession/ws/SubmittedVariantsRestControllerTest.java +++ b/eva-accession-ws/src/test/java/uk/ac/ebi/eva/accession/ws/SubmittedVariantsRestControllerTest.java @@ -17,10 +17,11 @@ */ package uk.ac.ebi.eva.accession.ws; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; +import jakarta.servlet.http.HttpServletResponse; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.Mockito; import org.springframework.beans.factory.annotation.Autowired; @@ -36,7 +37,7 @@ import org.springframework.http.ResponseEntity; import org.springframework.mock.web.MockHttpServletResponse; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import org.springframework.web.server.ResponseStatusException; import uk.ac.ebi.ampt2d.commons.accession.core.exceptions.AccessionCouldNotBeGeneratedException; import uk.ac.ebi.ampt2d.commons.accession.core.exceptions.AccessionDeprecatedException; @@ -46,19 +47,19 @@ import uk.ac.ebi.ampt2d.commons.accession.core.models.GetOrCreateAccessionWrapper; import uk.ac.ebi.ampt2d.commons.accession.rest.controllers.BasicRestController; import uk.ac.ebi.ampt2d.commons.accession.rest.dto.AccessionResponseDTO; - +import uk.ac.ebi.eva.accession.core.configuration.nonhuman.SubmittedVariantAccessioningConfiguration; import uk.ac.ebi.eva.accession.core.contigalias.ContigAliasService; import uk.ac.ebi.eva.accession.core.model.ISubmittedVariant; import uk.ac.ebi.eva.accession.core.model.SubmittedVariant; -import uk.ac.ebi.eva.accession.core.service.nonhuman.SubmittedVariantAccessioningService; -import uk.ac.ebi.eva.accession.core.configuration.nonhuman.SubmittedVariantAccessioningConfiguration; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpSubmittedVariantEntity; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpSubmittedVariantOperationEntity; -import uk.ac.ebi.eva.accession.core.repository.nonhuman.eva.SubmittedVariantAccessioningRepository; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantOperationEntity; +import uk.ac.ebi.eva.accession.core.repository.nonhuman.eva.SubmittedVariantAccessioningRepository; +import uk.ac.ebi.eva.accession.core.service.nonhuman.SubmittedVariantAccessioningService; import uk.ac.ebi.eva.accession.core.service.nonhuman.dbsnp.DbsnpSubmittedVariantInactiveService; import uk.ac.ebi.eva.accession.core.service.nonhuman.dbsnp.DbsnpSubmittedVariantMonotonicAccessioningService; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.ws.dto.BeaconAlleleRequest; import uk.ac.ebi.eva.accession.ws.dto.BeaconAlleleResponse; import uk.ac.ebi.eva.accession.ws.rest.SubmittedVariantsRestController; @@ -66,18 +67,17 @@ import uk.ac.ebi.eva.accession.ws.test.NoContigTranslationArgumentMatcher; import uk.ac.ebi.eva.commons.core.models.contigalias.ContigNamingConvention; -import javax.servlet.http.HttpServletResponse; import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.NoSuchElementException; import java.util.stream.Collectors; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertThrows; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.argThat; @@ -88,11 +88,11 @@ import static uk.ac.ebi.eva.accession.core.model.ISubmittedVariant.DEFAULT_SUPPORTED_BY_EVIDENCE; import static uk.ac.ebi.eva.accession.core.model.ISubmittedVariant.DEFAULT_VALIDATED; -@RunWith(SpringRunner.class) +@ExtendWith(SpringExtension.class) @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) @Import({SubmittedVariantAccessioningConfiguration.class}) @TestPropertySource("classpath:accession-ws-test.properties") -public class SubmittedVariantsRestControllerTest { +public class SubmittedVariantsRestControllerTest extends MongoTestContainerHelper{ private static String TEST_APPLICATION_INSTANCE_ID = "test-application-instance-id"; private static final String URL = "/v1/submitted-variants/"; @@ -139,7 +139,7 @@ public class SubmittedVariantsRestControllerTest { private SubmittedVariant variant3; - @Before + @BeforeEach public void setUp() throws AccessionCouldNotBeGeneratedException { repository.deleteAll(); mongoTemplate.dropCollection(DbsnpSubmittedVariantEntity.class); @@ -155,8 +155,8 @@ public void setUp() throws AccessionCouldNotBeGeneratedException { SubmittedVariantsBeaconService mockSubmittedVariantsBeaconService = Mockito.spy(new SubmittedVariantsBeaconService(service)); Mockito.doThrow(new RuntimeException("Some unexpected error")).when(mockSubmittedVariantsBeaconService).queryBeacon(null, "alt", "ref", - "CHROM1", 1, "ref", - ContigNamingConvention.INSDC, false); + "CHROM1", 1, "ref", + ContigNamingConvention.INSDC, false); mockController = new SubmittedVariantsRestController(mockService, mockSubmittedVariantsBeaconService); setUpContigAliasMock(); @@ -166,20 +166,20 @@ private void setUpContigAliasMock() { NoContigTranslationArgumentMatcher contigMatcher = new NoContigTranslationArgumentMatcher(); when(contigAliasService.translateContigToInsdc(anyString(), anyString(), argThat(contigMatcher))) - .thenCallRealMethod(); + .thenCallRealMethod(); when(contigAliasService.translateContigToInsdc(anyString(), anyString(), eq(ContigNamingConvention.ENA_SEQUENCE_NAME))) - .then(invocation -> { - String contigName = invocation.getArgument(0); - if (contigName.endsWith(ENA_CONTIG_SUFFIX)) { - return contigName.substring(0, contigName.length() - ENA_CONTIG_SUFFIX.length()); - } - throw new NoSuchElementException("Tried to translate non-ENA contig using ENA naming convention"); - }); + .then(invocation -> { + String contigName = invocation.getArgument(0); + if (contigName.endsWith(ENA_CONTIG_SUFFIX)) { + return contigName.substring(0, contigName.length() - ENA_CONTIG_SUFFIX.length()); + } + throw new NoSuchElementException("Tried to translate non-ENA contig using ENA naming convention"); + }); when(contigAliasService.translateContigToInsdc(anyString(), anyString(), eq(ContigNamingConvention.UCSC))) .thenThrow(NoSuchElementException.class); when(contigAliasService.translateContigFromInsdc(anyString(), argThat(contigMatcher))) - .thenCallRealMethod(); + .thenCallRealMethod(); when(contigAliasService.translateContigFromInsdc(any(), eq(ContigNamingConvention.ENA_SEQUENCE_NAME))) .then(invocation -> { String contigName = invocation.getArgument(0); @@ -193,10 +193,10 @@ private void setUpContigAliasMock() { .thenCallRealMethod(); when(contigAliasService.createSubmittedVariantAccessionWrapperWithNewContig(any(), anyString())) - .thenCallRealMethod(); + .thenCallRealMethod(); } - @After + @AfterEach public void tearDown() { repository.deleteAll(); mongoTemplate.dropCollection(DbsnpSubmittedVariantEntity.class); @@ -293,12 +293,12 @@ public void testGetByIdFieldsSingleStudyPerRequest() { ISubmittedVariant variant = generatedAccession.getData(); ResponseEntity>> getVariantsResponse = controller.getByIdFields(variant.getReferenceSequenceAccession(), - variant.getContig(), - Collections.singletonList( - variant.getProjectAccession()), - variant.getStart(), variant.getReferenceAllele(), - variant.getAlternateAllele(), - ContigNamingConvention.INSDC); + variant.getContig(), + Collections.singletonList( + variant.getProjectAccession()), + variant.getStart(), variant.getReferenceAllele(), + variant.getAlternateAllele(), + ContigNamingConvention.INSDC); assertEquals(1, getVariantsResponse.getBody().size()); assertCreatedDateNotNull(getVariantsResponse.getBody()); @@ -309,15 +309,15 @@ public void testGetByIdFieldsSingleStudyPerRequest() { @Test public void testGetByIdFieldsMultipleStudiesPerRequest() { List multipleProjectAccessions = Arrays.asList(variant1.getProjectAccession(), - variant2.getProjectAccession(), - variant3.getProjectAccession()); + variant2.getProjectAccession(), + variant3.getProjectAccession()); ResponseEntity>> getVariantsResponse = controller.getByIdFields(variant2.getReferenceSequenceAccession(), - variant2.getContig(), multipleProjectAccessions, - variant2.getStart(), variant2.getReferenceAllele(), - variant2.getAlternateAllele(), - ContigNamingConvention.INSDC); + variant2.getContig(), multipleProjectAccessions, + variant2.getStart(), variant2.getReferenceAllele(), + variant2.getAlternateAllele(), + ContigNamingConvention.INSDC); assertEquals(2, getVariantsResponse.getBody().size()); assertCreatedDateNotNull(getVariantsResponse.getBody()); @@ -331,12 +331,12 @@ public void testGetByIdFields_withContigTranslation() { ISubmittedVariant variant = generatedAccession.getData(); ResponseEntity>> getVariantsResponse = controller.getByIdFields(variant.getReferenceSequenceAccession(), - variant.getContig() + ENA_CONTIG_SUFFIX, - Collections.singletonList( - variant.getProjectAccession()), - variant.getStart(), variant.getReferenceAllele(), - variant.getAlternateAllele(), - ContigNamingConvention.ENA_SEQUENCE_NAME); + variant.getContig() + ENA_CONTIG_SUFFIX, + Collections.singletonList( + variant.getProjectAccession()), + variant.getStart(), variant.getReferenceAllele(), + variant.getAlternateAllele(), + ContigNamingConvention.ENA_SEQUENCE_NAME); assertEquals(1, getVariantsResponse.getBody().size()); assertCreatedDateNotNull(getVariantsResponse.getBody()); @@ -348,13 +348,13 @@ public void testGetByIdFields_withContigTranslation() { @Test public void testGetByIdFields_withWrongContigTranslation() { assertThrows(ResponseStatusException.class, - () -> controller.getByIdFields(variant1.getReferenceSequenceAccession(), - variant1.getContig(), - Collections.singletonList(variant1.getProjectAccession()), - variant1.getStart(), - variant1.getReferenceAllele(), - variant1.getAlternateAllele(), - ContigNamingConvention.UCSC)); + () -> controller.getByIdFields(variant1.getReferenceSequenceAccession(), + variant1.getContig(), + Collections.singletonList(variant1.getProjectAccession()), + variant1.getStart(), + variant1.getReferenceAllele(), + variant1.getAlternateAllele(), + ContigNamingConvention.UCSC)); } @Test @@ -382,8 +382,8 @@ public void testDoesVariantExistFoundExistingVariantsSingleStudyPerRequest() { @Test public void testDoesVariantExistFoundExistingVariantsMultipleStudiesPerRequest() { List multipleProjectAccessions = Arrays.asList(variant1.getProjectAccession(), - variant2.getProjectAccession(), - variant3.getProjectAccession()); + variant2.getProjectAccession(), + variant3.getProjectAccession()); for (AccessionWrapper generatedAccession : generatedAccessions) { ISubmittedVariant variant = generatedAccession.getData(); @@ -435,7 +435,7 @@ public void testDoesVariantExistWith400Error() { assertEquals(HttpServletResponse.SC_BAD_REQUEST, response.getStatus()); assertEquals(HttpServletResponse.SC_BAD_REQUEST, beaconAlleleResponse.getError().getErrorCode()); assertEquals("Please provide a positive number as start position", - beaconAlleleResponse.getError().getErrorMessage()); + beaconAlleleResponse.getError().getErrorMessage()); BeaconAlleleRequest embeddedRequestObject = beaconAlleleResponse.getAlleleRequest(); assertEquals(variant1.getReferenceSequenceAccession(), embeddedRequestObject.getAssemblyId()); @@ -449,7 +449,7 @@ public void testDoesVariantExistWith400Error() { public void testDoesVariantExistWith500Error() { HttpServletResponse response = new MockHttpServletResponse(); BeaconAlleleResponse beaconAlleleResponse = mockController.doesVariantExist( - "asm", "CHROM1" + ENA_CONTIG_SUFFIX, null,1, "ref", "alt", response); + "asm", "CHROM1" + ENA_CONTIG_SUFFIX, null, 1, "ref", "alt", response); assertEquals(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, response.getStatus()); assertEquals(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, beaconAlleleResponse.getError().getErrorCode()); @@ -465,26 +465,26 @@ public void testDoesVariantExistWith500Error() { @Test public void testGetRedirectionForMergedVariants() throws AccessionCouldNotBeGeneratedException, AccessionMergedException, AccessionDoesNotExistException, - AccessionDeprecatedException { + AccessionDeprecatedException { // given Long CLUSTERED_VARIANT = null; SubmittedVariant variant1 = new SubmittedVariant("ASMACC01", 2000, "PROJACC01", "CHROM1", 1234, "REF", "ALT", - CLUSTERED_VARIANT); + CLUSTERED_VARIANT); SubmittedVariant variant2 = new SubmittedVariant("ASMACC02", 2000, "PROJACC02", "CHROM2", 1234, "REF", "ALT", - CLUSTERED_VARIANT); + CLUSTERED_VARIANT); List> accessions = service.getOrCreate( Arrays.asList(variant1, variant2), TEST_APPLICATION_INSTANCE_ID); Long outdatedAccession = accessions.get(0).getAccession(); Long currentAccession = accessions.get(1).getAccession(); service.merge(outdatedAccession, - currentAccession, - "Just for testing the endpoint, let's pretend the variants are equivalent"); + currentAccession, + "Just for testing the endpoint, let's pretend the variants are equivalent"); // when String getVariantsUrl = URL + outdatedAccession; ResponseEntity firstResponse = testRestTemplate.exchange(getVariantsUrl, HttpMethod.GET, null, - String.class); + String.class); // then assertEquals(HttpStatus.MOVED_PERMANENTLY, firstResponse.getStatusCode()); @@ -507,7 +507,7 @@ public void testGetRedirectionForMergedVariants() /** * If there is a variant with operations of several types (e.g. UPDATED and MERGED) the MERGED event should take * priority and the endpoint should return a redirection, **even if the MERGED event is not the last one**. - * + *

* Example dbsnp variant: ss825691104. It was declustered from rs796064771 and merged into ss825691103 at the same * time. */ @@ -517,33 +517,33 @@ public void testGetRedirectionForMergedAndUpdatedVariants() // given Long CLUSTERED_VARIANT = null; SubmittedVariant variant1 = new SubmittedVariant("ASMACC01", 2000, "PROJACC01", "CHROM1", 1234, "REF", "ALT", - CLUSTERED_VARIANT); + CLUSTERED_VARIANT); Long outdatedAccession = 1L; SubmittedVariantEntity submittedVariantEntity1 = new SubmittedVariantEntity(outdatedAccession, "hash-100", - variant1, 1); + variant1, 1); Long currentAccession = 2L; SubmittedVariant variant2 = new SubmittedVariant("ASMACC02", 2000, "PROJACC02", "CHROM2", 1234, "REF", "ALT", - CLUSTERED_VARIANT); + CLUSTERED_VARIANT); SubmittedVariantEntity submittedVariantEntity2 = new SubmittedVariantEntity(currentAccession, "hash-200", - variant2, 1); + variant2, 1); mongoTemplate.insert(Arrays.asList(submittedVariantEntity1, submittedVariantEntity2), - DbsnpSubmittedVariantEntity.class); + DbsnpSubmittedVariantEntity.class); dbsnpService.merge(outdatedAccession, - currentAccession, - "Just for testing the endpoint, let's pretend the variants are equivalent"); + currentAccession, + "Just for testing the endpoint, let's pretend the variants are equivalent"); SubmittedVariant updatedVariant = new SubmittedVariant(variant1); updatedVariant.setContig("contig_2"); dbsnpInactiveService.update(new DbsnpSubmittedVariantEntity(outdatedAccession, "hash-300", updatedVariant, 1), - "update a merged variant"); + "update a merged variant"); // when String getVariantsUrl = URL + outdatedAccession; ResponseEntity firstResponse = testRestTemplate.exchange(getVariantsUrl, HttpMethod.GET, null, - String.class); + String.class); // then assertEquals(HttpStatus.MOVED_PERMANENTLY, firstResponse.getStatusCode()); @@ -566,10 +566,10 @@ public void testGetRedirectionForMergedAndUpdatedVariants() @Test public void testGetDeprecatedEvaSubmittedVariant() throws AccessionMergedException, AccessionDoesNotExistException, - AccessionDeprecatedException { + AccessionDeprecatedException { // given Long accession = generatedAccessions.stream().filter(wrapper -> wrapper.getData().equals(variant1)) - .findFirst().get().getAccession(); + .findFirst().get().getAccession(); service.deprecate(accession, "deprecated for testing"); String getVariantUrl = URL + accession; @@ -590,18 +590,18 @@ public void testGetDeprecatedDbsnpSubmittedVariant() // given Long CLUSTERED_VARIANT = null; SubmittedVariant variant1 = new SubmittedVariant("ASMACC01", 2000, "PROJACC01", "CHROM1", 1234, "REF", "ALT", - CLUSTERED_VARIANT); + CLUSTERED_VARIANT); Long deprecatedAccession = 1L; SubmittedVariantEntity submittedVariantEntity1 = new SubmittedVariantEntity(deprecatedAccession, "hash-100", - variant1, 1); + variant1, 1); Long otherAccession = 2L; SubmittedVariant variant2 = new SubmittedVariant("ASMACC02", 2000, "PROJACC02", "CHROM2", 1234, "REF", "ALT", - CLUSTERED_VARIANT); + CLUSTERED_VARIANT); SubmittedVariantEntity submittedVariantEntity2 = new SubmittedVariantEntity(otherAccession, "hash-200", - variant2, 1); + variant2, 1); mongoTemplate.insert(Arrays.asList(submittedVariantEntity1, submittedVariantEntity2), - DbsnpSubmittedVariantEntity.class); + DbsnpSubmittedVariantEntity.class); service.deprecate(deprecatedAccession, "deprecated for testing"); String getVariantUrl = URL + deprecatedAccession; diff --git a/eva-accession-ws/src/test/java/uk/ac/ebi/eva/accession/ws/test/MongoTestConfiguration.java b/eva-accession-ws/src/test/java/uk/ac/ebi/eva/accession/ws/test/MongoTestConfiguration.java new file mode 100644 index 000000000..5a4751e52 --- /dev/null +++ b/eva-accession-ws/src/test/java/uk/ac/ebi/eva/accession/ws/test/MongoTestConfiguration.java @@ -0,0 +1,105 @@ +/* + * Copyright 2021 EMBL - European Bioinformatics Institute + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package uk.ac.ebi.eva.accession.ws.test; + +import com.mongodb.ReadPreference; +import com.mongodb.WriteConcern; +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoClients; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.domain.EntityScan; +import org.springframework.boot.test.autoconfigure.data.mongo.AutoConfigureDataMongo; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Primary; +import org.springframework.data.mongodb.MongoDatabaseFactory; +import org.springframework.data.mongodb.config.EnableMongoAuditing; +import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.data.mongodb.core.SimpleMongoClientDatabaseFactory; +import org.springframework.data.mongodb.core.WriteResultChecking; +import org.springframework.data.mongodb.core.convert.DefaultDbRefResolver; +import org.springframework.data.mongodb.core.convert.DefaultMongoTypeMapper; +import org.springframework.data.mongodb.core.convert.MappingMongoConverter; +import org.springframework.data.mongodb.core.convert.MongoCustomConversions; +import org.springframework.data.mongodb.core.mapping.MongoMappingContext; +import org.springframework.data.mongodb.repository.config.EnableMongoRepositories; + +import java.util.Collections; + +@Configuration +@EntityScan(basePackages = {"uk.ac.ebi.eva.accession.core.persistence"}) +@EnableMongoRepositories(basePackages = { + "uk.ac.ebi.eva.accession.core.persistence", + "uk.ac.ebi.ampt2d.commons.accession.persistence.mongodb.repository"}) +@EnableMongoAuditing +@AutoConfigureDataMongo +public class MongoTestConfiguration { + + @Value("${spring.data.mongodb.uri}") + private String mongoUri; + + @Value("${spring.data.mongodb.database}") + private String database; + + @Value("${human.mongodb.database}") + private String humanDatabase; + + @Value("${mongodb.read-preference:primary}") + private String readPreference; + + @Bean + @Primary + public MongoClient mongoClient() { + return MongoClients.create(mongoUri); + } + + @Bean("primaryFactory") + @Primary + public MongoDatabaseFactory mongoDbFactory() { + return new SimpleMongoClientDatabaseFactory(mongoClient(), database); + } + + @Bean + @Primary + public MongoCustomConversions mongoCustomConversions() { + return new MongoCustomConversions(Collections.emptyList()); + } + + @Bean + @Primary + public MongoMappingContext mongoMappingContext(MongoCustomConversions conversions) { + MongoMappingContext context = new MongoMappingContext(); + context.setSimpleTypeHolder(conversions.getSimpleTypeHolder()); + context.setAutoIndexCreation(false); + return context; + } + + @Bean + @Primary + public MongoTemplate mongoTemplate(MongoMappingContext mongoMappingContext, + MongoCustomConversions conversions) { + MappingMongoConverter converter = new MappingMongoConverter( + new DefaultDbRefResolver(mongoDbFactory()), mongoMappingContext); + converter.setCustomConversions(conversions); + converter.setTypeMapper(new DefaultMongoTypeMapper(null)); + converter.afterPropertiesSet(); + MongoTemplate mongoTemplate = new MongoTemplate(mongoDbFactory(), converter); + mongoTemplate.setReadPreference(ReadPreference.valueOf(readPreference)); + mongoTemplate.setWriteConcern(WriteConcern.MAJORITY); + mongoTemplate.setWriteResultChecking(WriteResultChecking.EXCEPTION); + return mongoTemplate; + } +} diff --git a/eva-accession-ws/src/test/java/uk/ac/ebi/eva/accession/ws/test/TestConfiguration.java b/eva-accession-ws/src/test/java/uk/ac/ebi/eva/accession/ws/test/TestConfiguration.java index b4ede1ea3..998c549a4 100644 --- a/eva-accession-ws/src/test/java/uk/ac/ebi/eva/accession/ws/test/TestConfiguration.java +++ b/eva-accession-ws/src/test/java/uk/ac/ebi/eva/accession/ws/test/TestConfiguration.java @@ -20,13 +20,12 @@ import org.springframework.context.annotation.Configuration; import uk.ac.ebi.ampt2d.commons.accession.core.AccessionSaveMode; import uk.ac.ebi.ampt2d.commons.accession.hashing.SHA1HashingFunction; - +import uk.ac.ebi.eva.accession.core.generators.DbsnpMonotonicAccessionGenerator; import uk.ac.ebi.eva.accession.core.model.IClusteredVariant; import uk.ac.ebi.eva.accession.core.model.ISubmittedVariant; import uk.ac.ebi.eva.accession.core.service.nonhuman.dbsnp.DbsnpClusteredVariantAccessioningDatabaseService; -import uk.ac.ebi.eva.accession.core.generators.DbsnpMonotonicAccessionGenerator; -import uk.ac.ebi.eva.accession.core.service.nonhuman.dbsnp.DbsnpSubmittedVariantAccessioningDatabaseService; import uk.ac.ebi.eva.accession.core.service.nonhuman.dbsnp.DbsnpClusteredVariantMonotonicAccessioningService; +import uk.ac.ebi.eva.accession.core.service.nonhuman.dbsnp.DbsnpSubmittedVariantAccessioningDatabaseService; import uk.ac.ebi.eva.accession.core.service.nonhuman.dbsnp.DbsnpSubmittedVariantMonotonicAccessioningService; import uk.ac.ebi.eva.accession.core.summary.ClusteredVariantSummaryFunction; import uk.ac.ebi.eva.accession.core.summary.SubmittedVariantSummaryFunction; diff --git a/eva-accession-ws/src/test/java/uk/ac/ebi/eva/accession/ws/test/rule/FixSpringMongoDbRule.java b/eva-accession-ws/src/test/java/uk/ac/ebi/eva/accession/ws/test/rule/FixSpringMongoDbRule.java deleted file mode 100644 index b3e08f7a5..000000000 --- a/eva-accession-ws/src/test/java/uk/ac/ebi/eva/accession/ws/test/rule/FixSpringMongoDbRule.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * - * Copyright 2021 EMBL - European Bioinformatics Institute - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -package uk.ac.ebi.eva.accession.ws.test.rule; - -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfiguration; -import com.lordofthejars.nosqlunit.mongodb.SpringMongoDbRule; - - -/** - * Temporary fix until nosql unit rc-6 or final is released - */ -public class FixSpringMongoDbRule extends SpringMongoDbRule { - - public FixSpringMongoDbRule(MongoDbConfiguration mongoDbConfiguration) { - super(mongoDbConfiguration); - } - - public FixSpringMongoDbRule(MongoDbConfiguration mongoDbConfiguration, Object object) { - super(mongoDbConfiguration, object); - } - - @Override - public void close() { - // DO NOT CLOSE the connection (Spring will do it when destroying the context) - } -} diff --git a/eva-accession-ws/src/test/resources/accession-ws-test.properties b/eva-accession-ws/src/test/resources/accession-ws-test.properties index 3cf1bcdc1..82e3e9615 100644 --- a/eva-accession-ws/src/test/resources/accession-ws-test.properties +++ b/eva-accession-ws/src/test/resources/accession-ws-test.properties @@ -14,13 +14,10 @@ accessioning.monotonic.test-rs.nextBlockInterval=1000000000 # this will be replaced with a default value "localhost" when run locally # (due to the filtering attribute specified in this project's POM) # (OR) this variable can be overridden in Gitlab with "mongo" (Gitlab services are exposed under their own hostname) -spring.data.mongodb.uri=mongodb://|eva.mongo.host.test|:27017 spring.data.mongodb.database=eva-accession-ws-test-db # See https://github.com/spring-projects/spring-boot/wiki/Spring-Boot-2.1-Release-Notes#bean-overriding spring.main.allow-bean-definition-overriding=true mongodb.read-preference=primary - -human.mongodb.uri=mongodb://|eva.mongo.host.test|:27017 human.mongodb.database=eva-accession-ws-test-db-human eva.api.base-url=http://localhost diff --git a/eva-remapping-get-source/pom.xml b/eva-remapping-get-source/pom.xml index c9fc4a5b7..a01e9bc5c 100644 --- a/eva-remapping-get-source/pom.xml +++ b/eva-remapping-get-source/pom.xml @@ -2,12 +2,13 @@ + 4.0.0 + eva-accession uk.ac.ebi.eva - 0.6.57-SNAPSHOT + ${revision} - 4.0.0 eva-remapping-get-source jar @@ -24,8 +25,8 @@ uk.ac.ebi.eva eva-accession-core + ${revision} test-jar - ${project.version} test @@ -48,8 +49,18 @@ test - com.lordofthejars - nosqlunit-mongodb + org.springframework.boot + spring-boot-testcontainers + test + + + org.testcontainers + mongodb + test + + + org.testcontainers + junit-jupiter test @@ -90,8 +101,4 @@ - - localhost - - diff --git a/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/batch/io/DbsnpSubmittedVariantMongoReader.java b/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/batch/io/DbsnpSubmittedVariantMongoReader.java index c3e98cfcc..648edf5a7 100644 --- a/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/batch/io/DbsnpSubmittedVariantMongoReader.java +++ b/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/batch/io/DbsnpSubmittedVariantMongoReader.java @@ -15,7 +15,6 @@ */ package uk.ac.ebi.eva.remapping.source.batch.io; -import org.apache.commons.collections.CollectionUtils; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.data.mongodb.core.query.Criteria; import org.springframework.data.mongodb.core.query.Meta; @@ -39,14 +38,14 @@ public class DbsnpSubmittedVariantMongoReader extends MongoDbCursorItemReader projects, int taxonomy) { + List projects, int taxonomy) { setMongoTemplate(mongoTemplate); setTargetType(DbsnpSubmittedVariantEntity.class); Criteria criteria = where(REFERENCE_SEQUENCE_FIELD).is(assemblyAccession); criteria.and(REMAPPEDFROM_KEY).exists(false); - if (!CollectionUtils.isEmpty(projects)) { + if (projects != null && !projects.isEmpty()) { criteria.and(PROJECT_KEY).in(projects); } diff --git a/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/batch/io/EvaSubmittedVariantMongoReader.java b/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/batch/io/EvaSubmittedVariantMongoReader.java index 6e64de229..1c4707708 100644 --- a/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/batch/io/EvaSubmittedVariantMongoReader.java +++ b/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/batch/io/EvaSubmittedVariantMongoReader.java @@ -15,7 +15,6 @@ */ package uk.ac.ebi.eva.remapping.source.batch.io; -import org.apache.commons.collections.CollectionUtils; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.data.mongodb.core.query.Criteria; import org.springframework.data.mongodb.core.query.Meta; @@ -46,7 +45,7 @@ public EvaSubmittedVariantMongoReader(String assemblyAccession, MongoTemplate mo Criteria criteria = where(REFERENCE_SEQUENCE_FIELD).is(assemblyAccession); criteria.and(REMAPPEDFROM_KEY).exists(false); - if (!CollectionUtils.isEmpty(projects)) { + if (projects != null && !projects.isEmpty()) { criteria.and(PROJECT_KEY).in(projects); } diff --git a/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/batch/io/VariantContextWriter.java b/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/batch/io/VariantContextWriter.java index 52e7a03b9..a186d72d4 100644 --- a/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/batch/io/VariantContextWriter.java +++ b/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/batch/io/VariantContextWriter.java @@ -24,6 +24,7 @@ import htsjdk.variant.vcf.VCFInfoHeaderLine; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.batch.item.Chunk; import org.springframework.batch.item.ExecutionContext; import org.springframework.batch.item.ItemStreamException; import org.springframework.batch.item.ItemStreamWriter; @@ -31,7 +32,6 @@ import java.io.File; import java.nio.file.Path; import java.util.HashSet; -import java.util.List; import java.util.Set; /** @@ -87,15 +87,15 @@ protected Set buildHeaderLines() { metaData.add(new VCFInfoHeaderLine(SS_HASH, 1, VCFHeaderLineType.String, "Hash (_id in MongoDB) of the Submitted Variant from which the remapped variant was created")); metaData.add(new VCFInfoHeaderLine(RS_KEY, 1, VCFHeaderLineType.String, - "RS ID where this SS ID is clustered")); + "RS ID where this SS ID is clustered")); metaData.add(new VCFInfoHeaderLine(BACKPROP_RS_KEY, 1, VCFHeaderLineType.String, - "RS ID that was backpropagated to this SS from another remapping")); + "RS ID that was backpropagated to this SS from another remapping")); metaData.add(new VCFInfoHeaderLine(PROJECT_KEY, 1, VCFHeaderLineType.String, - "PROJECT ID associated with this SS ID")); + "PROJECT ID associated with this SS ID")); metaData.add(new VCFInfoHeaderLine(TAXONOMY, 1, VCFHeaderLineType.String, - "TAXONOMY ID associated with this SS ID")); + "TAXONOMY ID associated with this SS ID")); metaData.add(new VCFInfoHeaderLine(CREATED_DATE, 1, VCFHeaderLineType.String, - "Date when the SS ID was created")); + "Date when the SS ID was created")); return metaData; } @@ -104,7 +104,7 @@ private String getReferenceAssemblyLine() { } @Override - public void write(List variantContexts) throws Exception { + public void write(Chunk variantContexts) throws Exception { for (VariantContext variantContext : variantContexts) { writer.add(variantContext); } diff --git a/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/batch/processors/SubmittedVariantToVariantContextProcessor.java b/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/batch/processors/SubmittedVariantToVariantContextProcessor.java index df8e12025..44f08ee8a 100644 --- a/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/batch/processors/SubmittedVariantToVariantContextProcessor.java +++ b/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/batch/processors/SubmittedVariantToVariantContextProcessor.java @@ -19,7 +19,6 @@ import htsjdk.variant.variantcontext.VariantContext; import htsjdk.variant.variantcontext.VariantContextBuilder; import org.springframework.batch.item.ItemProcessor; - import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; import java.util.HashMap; @@ -27,9 +26,9 @@ import static uk.ac.ebi.eva.remapping.source.batch.io.VariantContextWriter.BACKPROP_RS_KEY; import static uk.ac.ebi.eva.remapping.source.batch.io.VariantContextWriter.CREATED_DATE; +import static uk.ac.ebi.eva.remapping.source.batch.io.VariantContextWriter.PROJECT_KEY; import static uk.ac.ebi.eva.remapping.source.batch.io.VariantContextWriter.RS_KEY; import static uk.ac.ebi.eva.remapping.source.batch.io.VariantContextWriter.SS_HASH; -import static uk.ac.ebi.eva.remapping.source.batch.io.VariantContextWriter.PROJECT_KEY; import static uk.ac.ebi.eva.remapping.source.batch.io.VariantContextWriter.TAXONOMY; /** diff --git a/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/configuration/RunnerConfiguration.java b/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/configuration/RunnerConfiguration.java index 1542410ed..01d53cd52 100644 --- a/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/configuration/RunnerConfiguration.java +++ b/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/configuration/RunnerConfiguration.java @@ -16,16 +16,10 @@ */ package uk.ac.ebi.eva.remapping.source.configuration; -import org.springframework.batch.core.configuration.annotation.BatchConfigurer; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; - -import uk.ac.ebi.eva.commons.batch.configuration.SpringBoot1CompatibilityConfiguration; import uk.ac.ebi.eva.commons.batch.job.JobExecutionApplicationListener; -import javax.persistence.EntityManagerFactory; -import javax.sql.DataSource; - @Configuration public class RunnerConfiguration { @@ -33,11 +27,4 @@ public class RunnerConfiguration { public JobExecutionApplicationListener jobExecutionApplicationListener() { return new JobExecutionApplicationListener(); } - - @Bean - public BatchConfigurer configurer(DataSource dataSource, EntityManagerFactory entityManagerFactory) - throws Exception { - return SpringBoot1CompatibilityConfiguration.getSpringBoot1CompatibleBatchConfigurer(dataSource, - entityManagerFactory); - } } \ No newline at end of file diff --git a/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/configuration/batch/io/VariantContextWriterConfiguration.java b/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/configuration/batch/io/VariantContextWriterConfiguration.java index c96348cab..e751d5d57 100644 --- a/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/configuration/batch/io/VariantContextWriterConfiguration.java +++ b/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/configuration/batch/io/VariantContextWriterConfiguration.java @@ -18,11 +18,10 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; - import uk.ac.ebi.eva.remapping.source.batch.io.VariantContextWriter; +import uk.ac.ebi.eva.remapping.source.configuration.BeanNames; import uk.ac.ebi.eva.remapping.source.parameters.InputParameters; import uk.ac.ebi.eva.remapping.source.parameters.ReportPathResolver; -import uk.ac.ebi.eva.remapping.source.configuration.BeanNames; import java.nio.file.Path; diff --git a/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/configuration/batch/jobs/ExportSubmittedVariantsJobConfiguration.java b/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/configuration/batch/jobs/ExportSubmittedVariantsJobConfiguration.java index ecd852e4e..c6a982d6a 100644 --- a/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/configuration/batch/jobs/ExportSubmittedVariantsJobConfiguration.java +++ b/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/configuration/batch/jobs/ExportSubmittedVariantsJobConfiguration.java @@ -19,16 +19,16 @@ import org.springframework.batch.core.Job; import org.springframework.batch.core.Step; import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; -import org.springframework.batch.core.configuration.annotation.JobBuilderFactory; +import org.springframework.batch.core.job.builder.JobBuilder; import org.springframework.batch.core.launch.support.RunIdIncrementer; +import org.springframework.batch.core.repository.JobRepository; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Import; - -import uk.ac.ebi.eva.remapping.source.configuration.batch.steps.ExportSubmittedVariantsStepConfiguration; import uk.ac.ebi.eva.remapping.source.configuration.BeanNames; +import uk.ac.ebi.eva.remapping.source.configuration.batch.steps.ExportSubmittedVariantsStepConfiguration; @Configuration @EnableBatchProcessing @@ -37,14 +37,14 @@ public class ExportSubmittedVariantsJobConfiguration { @Bean(BeanNames.EXPORT_SUBMITTED_VARIANTS_JOB) public Job accessionReleaseJob( - JobBuilderFactory jobBuilderFactory, + JobRepository jobRepository, @Autowired @Qualifier(BeanNames.EXPORT_EVA_SUBMITTED_VARIANTS_STEP) Step exportEvaSubmittedVariantsStep, @Autowired @Qualifier(BeanNames.EXPORT_DBSNP_SUBMITTED_VARIANTS_STEP) Step exportDbsnpSubmittedVariantsStep ) { - return jobBuilderFactory.get(BeanNames.EXPORT_SUBMITTED_VARIANTS_JOB) - .incrementer(new RunIdIncrementer()) - .start(exportEvaSubmittedVariantsStep) - .next(exportDbsnpSubmittedVariantsStep) - .build(); + return new JobBuilder(BeanNames.EXPORT_SUBMITTED_VARIANTS_JOB, jobRepository) + .incrementer(new RunIdIncrementer()) + .start(exportEvaSubmittedVariantsStep) + .next(exportDbsnpSubmittedVariantsStep) + .build(); } } diff --git a/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/configuration/batch/listeners/ListenersConfiguration.java b/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/configuration/batch/listeners/ListenersConfiguration.java index 4078594dd..47563cb45 100644 --- a/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/configuration/batch/listeners/ListenersConfiguration.java +++ b/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/configuration/batch/listeners/ListenersConfiguration.java @@ -20,12 +20,11 @@ import org.springframework.batch.core.listener.StepListenerSupport; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; - -import uk.ac.ebi.eva.remapping.source.batch.listeners.ExcludeVariantsListener; import uk.ac.ebi.eva.accession.core.batch.listeners.GenericProgressListener; -import uk.ac.ebi.eva.remapping.source.parameters.InputParameters; import uk.ac.ebi.eva.commons.core.models.pipeline.Variant; +import uk.ac.ebi.eva.remapping.source.batch.listeners.ExcludeVariantsListener; import uk.ac.ebi.eva.remapping.source.configuration.BeanNames; +import uk.ac.ebi.eva.remapping.source.parameters.InputParameters; @Configuration public class ListenersConfiguration { diff --git a/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/configuration/batch/steps/ExportSubmittedVariantsStepConfiguration.java b/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/configuration/batch/steps/ExportSubmittedVariantsStepConfiguration.java index be62e34e9..14af3acba 100644 --- a/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/configuration/batch/steps/ExportSubmittedVariantsStepConfiguration.java +++ b/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/configuration/batch/steps/ExportSubmittedVariantsStepConfiguration.java @@ -18,7 +18,8 @@ import htsjdk.variant.variantcontext.VariantContext; import org.springframework.batch.core.Step; import org.springframework.batch.core.StepExecutionListener; -import org.springframework.batch.core.configuration.annotation.StepBuilderFactory; +import org.springframework.batch.core.repository.JobRepository; +import org.springframework.batch.core.step.builder.StepBuilder; import org.springframework.batch.core.step.tasklet.TaskletStep; import org.springframework.batch.item.ItemProcessor; import org.springframework.batch.item.ItemReader; @@ -29,16 +30,16 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Import; - +import org.springframework.transaction.PlatformTransactionManager; +import uk.ac.ebi.eva.accession.core.batch.policies.IllegalStartSkipPolicy; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpSubmittedVariantEntity; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; -import uk.ac.ebi.eva.accession.core.batch.policies.IllegalStartSkipPolicy; +import uk.ac.ebi.eva.remapping.source.configuration.BeanNames; import uk.ac.ebi.eva.remapping.source.configuration.batch.io.SubmittedVariantMongoReaderConfiguration; import uk.ac.ebi.eva.remapping.source.configuration.batch.io.VariantContextWriterConfiguration; import uk.ac.ebi.eva.remapping.source.configuration.batch.listeners.ListenersConfiguration; import uk.ac.ebi.eva.remapping.source.configuration.batch.policies.PoliciesConfiguration; import uk.ac.ebi.eva.remapping.source.configuration.batch.processors.SubmittedVariantsProcessorConfiguration; -import uk.ac.ebi.eva.remapping.source.configuration.BeanNames; @Configuration @Import({SubmittedVariantMongoReaderConfiguration.class, @@ -50,7 +51,7 @@ public class ExportSubmittedVariantsStepConfiguration { @Bean(BeanNames.EXPORT_EVA_SUBMITTED_VARIANTS_STEP) public Step exportEvaSubmittedVariantsStep( - StepBuilderFactory stepBuilderFactory, + JobRepository jobRepository, PlatformTransactionManager transactionManager, SimpleCompletionPolicy chunkSizeCompletionPolicy, @Autowired @Qualifier(BeanNames.EVA_SUBMITTED_VARIANT_READER) ItemReader variantReader, @Autowired @Qualifier(BeanNames.SUBMITTED_VARIANT_PROCESSOR) ItemProcessor variantProcessor, @@ -58,8 +59,8 @@ public Step exportEvaSubmittedVariantsStep( @Autowired @Qualifier(BeanNames.PROGRESS_LISTENER) StepExecutionListener progressListener, @Autowired @Qualifier(BeanNames.EXCLUDE_VARIANTS_LISTENER) StepExecutionListener excludeVariantsListener, @Autowired IllegalStartSkipPolicy illegalStartSkipPolicy) { - TaskletStep step = stepBuilderFactory.get(BeanNames.EXPORT_EVA_SUBMITTED_VARIANTS_STEP) - .chunk(chunkSizeCompletionPolicy) + TaskletStep step = new StepBuilder(BeanNames.EXPORT_EVA_SUBMITTED_VARIANTS_STEP, jobRepository) + .chunk(chunkSizeCompletionPolicy, transactionManager) .reader(variantReader) .processor(variantProcessor) .writer(accessionWriter) @@ -73,7 +74,7 @@ public Step exportEvaSubmittedVariantsStep( @Bean(BeanNames.EXPORT_DBSNP_SUBMITTED_VARIANTS_STEP) public Step exportDbsnpSubmittedVariantsStep( - StepBuilderFactory stepBuilderFactory, + JobRepository jobRepository, PlatformTransactionManager transactionManager, SimpleCompletionPolicy chunkSizeCompletionPolicy, @Autowired @Qualifier(BeanNames.DBSNP_SUBMITTED_VARIANT_READER) ItemReader variantReader, @Autowired @Qualifier(BeanNames.SUBMITTED_VARIANT_PROCESSOR) ItemProcessor variantProcessor, @@ -81,8 +82,8 @@ public Step exportDbsnpSubmittedVariantsStep( @Autowired @Qualifier(BeanNames.PROGRESS_LISTENER) StepExecutionListener progressListener, @Autowired @Qualifier(BeanNames.EXCLUDE_VARIANTS_LISTENER) StepExecutionListener excludeVariantsListener, @Autowired IllegalStartSkipPolicy illegalStartSkipPolicy) { - TaskletStep step = stepBuilderFactory.get(BeanNames.EXPORT_DBSNP_SUBMITTED_VARIANTS_STEP) - .chunk(chunkSizeCompletionPolicy) + TaskletStep step = new StepBuilder(BeanNames.EXPORT_DBSNP_SUBMITTED_VARIANTS_STEP, jobRepository) + .chunk(chunkSizeCompletionPolicy, transactionManager) .reader(variantReader) .processor(variantProcessor) .writer(accessionWriter) diff --git a/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/parameters/InputParameters.java b/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/parameters/InputParameters.java index f64e8b77a..4f2d34406 100644 --- a/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/parameters/InputParameters.java +++ b/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/parameters/InputParameters.java @@ -16,7 +16,6 @@ package uk.ac.ebi.eva.remapping.source.parameters; -import org.apache.commons.collections.CollectionUtils; import org.springframework.batch.core.JobParameters; import org.springframework.batch.core.JobParametersBuilder; @@ -46,8 +45,8 @@ public JobParameters toJobParameters() { .addString("fasta", fasta) .addString("assemblyReportUrl", assemblyReportUrl) .addString("outputFolder", outputFolder) - .addString("projects", CollectionUtils.isEmpty(projects) ? "" : String.join(",", projects)) - .addLong("taxonomy", (long)taxonomy) + .addString("projects", (projects == null || projects.isEmpty()) ? "" : String.join(",", projects)) + .addLong("taxonomy", (long) taxonomy) .toJobParameters(); } diff --git a/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/runner/AccessionRemappingJobLauncherCommandLineRunner.java b/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/runner/AccessionRemappingJobLauncherCommandLineRunner.java index 70e6ea907..ee20d70ae 100644 --- a/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/runner/AccessionRemappingJobLauncherCommandLineRunner.java +++ b/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/runner/AccessionRemappingJobLauncherCommandLineRunner.java @@ -24,7 +24,6 @@ import org.springframework.batch.core.JobParametersInvalidException; import org.springframework.batch.core.explore.JobExplorer; import org.springframework.batch.core.launch.JobLauncher; -import org.springframework.batch.core.launch.JobParametersNotFoundException; import org.springframework.batch.core.repository.JobExecutionAlreadyRunningException; import org.springframework.batch.core.repository.JobInstanceAlreadyCompleteException; import org.springframework.batch.core.repository.JobRepository; @@ -32,24 +31,23 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.ExitCodeGenerator; -import org.springframework.boot.autoconfigure.batch.JobLauncherCommandLineRunner; +import org.springframework.boot.autoconfigure.batch.JobLauncherApplicationRunner; import org.springframework.context.ApplicationEventPublisherAware; import org.springframework.stereotype.Component; import org.springframework.util.PatternMatchUtils; - import uk.ac.ebi.eva.accession.core.runner.CommandLineRunnerUtils; -import uk.ac.ebi.eva.remapping.source.parameters.InputParameters; import uk.ac.ebi.eva.commons.batch.exception.NoJobToExecuteException; import uk.ac.ebi.eva.commons.batch.exception.NoParametersHaveBeenPassedException; import uk.ac.ebi.eva.commons.batch.exception.NoPreviousJobExecutionException; import uk.ac.ebi.eva.commons.batch.exception.UnknownJobException; import uk.ac.ebi.eva.commons.batch.job.JobExecutionApplicationListener; import uk.ac.ebi.eva.commons.batch.job.JobStatusManager; +import uk.ac.ebi.eva.remapping.source.parameters.InputParameters; import java.util.Collection; @Component -public class AccessionRemappingJobLauncherCommandLineRunner extends JobLauncherCommandLineRunner implements +public class AccessionRemappingJobLauncherCommandLineRunner extends JobLauncherApplicationRunner implements ApplicationEventPublisherAware, ExitCodeGenerator { private static final Logger logger = LoggerFactory.getLogger(AccessionRemappingJobLauncherCommandLineRunner.class); @@ -92,9 +90,9 @@ public void setJobs(Collection jobs) { } @Override - public void setJobNames(String jobName) { + public void setJobName(String jobName) { this.jobName = jobName; - super.setJobNames(jobName); + super.setJobName(jobName); } @Override @@ -119,17 +117,16 @@ public void run(String... args) throws JobExecutionException { if (inputParameters.isForceRestart()) { JobExecution previousJobExecution = CommandLineRunnerUtils.getLastJobExecution(jobName, jobExplorer, - jobParameters); + jobParameters); CommandLineRunnerUtils.markPreviousJobAsFailed(jobName, jobRepository, previousJobExecution != null ? previousJobExecution.getJobParameters() : jobParameters); - } - else { + } else { jobParameters = CommandLineRunnerUtils.addRunIDToJobParameters(jobName, jobExplorer, jobParameters); } launchJob(jobParameters); } catch (NoJobToExecuteException | NoParametersHaveBeenPassedException | NoPreviousJobExecutionException - | UnknownJobException | JobParametersInvalidException | JobExecutionAlreadyRunningException e) { + | UnknownJobException | JobParametersInvalidException | JobExecutionAlreadyRunningException e) { logger.error(e.getMessage()); logger.debug("Error trace", e); abnormalExit = true; @@ -151,8 +148,7 @@ private void launchJob(JobParameters jobParameters) throws JobExecutionException @Override protected void execute(Job job, JobParameters jobParameters) throws JobExecutionAlreadyRunningException, - JobRestartException, JobInstanceAlreadyCompleteException, JobParametersInvalidException, - JobParametersNotFoundException { + JobRestartException, JobInstanceAlreadyCompleteException, JobParametersInvalidException { logger.info("Running job '" + jobName + "' with parameters: " + jobParameters); super.execute(job, jobParameters); } diff --git a/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/batch/io/DbsnpSubmittedVariantMongoReaderTest.java b/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/batch/io/DbsnpSubmittedVariantMongoReaderTest.java index 83292677e..c397c49ab 100644 --- a/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/batch/io/DbsnpSubmittedVariantMongoReaderTest.java +++ b/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/batch/io/DbsnpSubmittedVariantMongoReaderTest.java @@ -16,26 +16,22 @@ package uk.ac.ebi.eva.remapping.source.batch.io; -import com.lordofthejars.nosqlunit.annotation.UsingDataSet; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.batch.item.ExecutionContext; import org.springframework.batch.item.ItemReader; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.ApplicationContext; +import org.springframework.core.io.ResourceLoader; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; - +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.MongoConfiguration; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpSubmittedVariantEntity; -import uk.ac.ebi.eva.accession.core.test.rule.FixSpringMongoDbRule; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; +import uk.ac.ebi.eva.accession.core.utils.MongoTestDataLoader; import uk.ac.ebi.eva.remapping.source.test.configuration.MongoTestConfiguration; import java.util.ArrayList; @@ -43,43 +39,38 @@ import java.util.Collections; import java.util.List; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; -@RunWith(SpringRunner.class) +@ExtendWith(SpringExtension.class) @TestPropertySource("classpath:application.properties") -@UsingDataSet(locations = { - "/test-data/dbsnpSubmittedVariantEntity.json" -}) @ContextConfiguration(classes = {MongoConfiguration.class, MongoTestConfiguration.class}) -public class DbsnpSubmittedVariantMongoReaderTest { - - private static final String TEST_DB = "test-db"; +public class DbsnpSubmittedVariantMongoReaderTest extends MongoTestContainerHelper { private static final String ASSEMBLY = "GCA_000409795.2"; @Autowired private MongoTemplate mongoTemplate; - //Required by nosql-unit @Autowired - private ApplicationContext applicationContext; - - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName(TEST_DB).build()); + private ResourceLoader resourceLoader; private DbsnpSubmittedVariantMongoReader reader; - @Before + @BeforeEach public void setUp() { + mongoTemplate.getDb().drop(); + + new MongoTestDataLoader(mongoTemplate, resourceLoader).load("/test-data/dbsnpSubmittedVariantEntity.json"); + ExecutionContext executionContext = new ExecutionContext(); reader = new DbsnpSubmittedVariantMongoReader(ASSEMBLY, mongoTemplate, null, 0); reader.open(executionContext); } - @After + @AfterEach public void tearDown() { reader.close(); + mongoTemplate.getDb().drop(); } @Test diff --git a/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/batch/io/EvaSubmittedVariantMongoReaderTest.java b/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/batch/io/EvaSubmittedVariantMongoReaderTest.java index 44c1005e1..6925f93e2 100644 --- a/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/batch/io/EvaSubmittedVariantMongoReaderTest.java +++ b/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/batch/io/EvaSubmittedVariantMongoReaderTest.java @@ -16,69 +16,59 @@ package uk.ac.ebi.eva.remapping.source.batch.io; -import com.lordofthejars.nosqlunit.annotation.UsingDataSet; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.batch.item.ExecutionContext; import org.springframework.batch.item.ItemReader; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.ApplicationContext; +import org.springframework.core.io.ResourceLoader; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; - +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.MongoConfiguration; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; -import uk.ac.ebi.eva.accession.core.test.rule.FixSpringMongoDbRule; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; +import uk.ac.ebi.eva.accession.core.utils.MongoTestDataLoader; import uk.ac.ebi.eva.remapping.source.test.configuration.MongoTestConfiguration; import java.util.ArrayList; import java.util.Collections; import java.util.List; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; -@RunWith(SpringRunner.class) +@ExtendWith(SpringExtension.class) @TestPropertySource("classpath:application.properties") -@UsingDataSet(locations = { - "/test-data/submittedVariantEntity.json" -}) @ContextConfiguration(classes = {MongoConfiguration.class, MongoTestConfiguration.class}) -public class EvaSubmittedVariantMongoReaderTest { - - private static final String TEST_DB = "test-db"; - +public class EvaSubmittedVariantMongoReaderTest extends MongoTestContainerHelper { private static final String ASSEMBLY = "GCA_000409795.2"; @Autowired private MongoTemplate mongoTemplate; - //Required by nosql-unit @Autowired - private ApplicationContext applicationContext; - - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName(TEST_DB).build()); + private ResourceLoader resourceLoader; private EvaSubmittedVariantMongoReader reader; - @Before + @BeforeEach public void setUp() { + mongoTemplate.getDb().drop(); + + new MongoTestDataLoader(mongoTemplate, resourceLoader).load("/test-data/submittedVariantEntity.json"); + ExecutionContext executionContext = new ExecutionContext(); reader = new EvaSubmittedVariantMongoReader(ASSEMBLY, mongoTemplate, null, 0); reader.open(executionContext); } - @After + @AfterEach public void tearDown() { reader.close(); + mongoTemplate.getDb().drop(); } @Test diff --git a/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/batch/io/VariantContextWriterTest.java b/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/batch/io/VariantContextWriterTest.java index 97ce1cfd3..d1294bb82 100644 --- a/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/batch/io/VariantContextWriterTest.java +++ b/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/batch/io/VariantContextWriterTest.java @@ -18,11 +18,10 @@ package uk.ac.ebi.eva.remapping.source.batch.io; import htsjdk.variant.variantcontext.VariantContext; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; - +import org.junit.jupiter.api.Test; +import org.springframework.batch.item.Chunk; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; +import uk.ac.ebi.eva.accession.core.utils.PipelineTemporaryFolderUtil; import uk.ac.ebi.eva.remapping.source.batch.processors.SubmittedVariantToVariantContextProcessor; import uk.ac.ebi.eva.remapping.source.parameters.ReportPathResolver; @@ -37,8 +36,8 @@ import java.util.stream.Collectors; import java.util.stream.Stream; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; public class VariantContextWriterTest { @@ -64,12 +63,11 @@ public class VariantContextWriterTest { public static final String SS_HASH = "hash1"; - @Rule - public TemporaryFolder temporaryFolder = new TemporaryFolder(); + public PipelineTemporaryFolderUtil temporaryFolderUtil = new PipelineTemporaryFolderUtil(); @Test public void basicWrite() throws Exception { - File outputFolder = temporaryFolder.newFolder(); + File outputFolder = temporaryFolderUtil.newFolder(); assertWriteVcf(outputFolder, buildVariant("1", 1000, "C", "A", 123L)); } @@ -84,7 +82,7 @@ private File assertWriteVcf(File outputFolder, SubmittedVariantEntity... variant List variantContexts = Stream.of(variants).map(variantToVariantContextProcessor::process).collect(Collectors.toList()); - writer.write(variantContexts); + writer.write(new Chunk<>(variantContexts)); writer.close(); @@ -106,9 +104,9 @@ private SubmittedVariantEntity buildVariant(String chr, long start, String ref, private SubmittedVariantEntity buildVariant(String chr, long start, String ref, String alt, Long rs, Long backpropRs, String project, int taxonomy) { SubmittedVariantEntity submittedVariantEntity = new SubmittedVariantEntity(1L, SS_HASH, REFERENCE_ASSEMBLY, - taxonomy, project, chr, start, ref, - alt, rs, false, false, false, false, - 1); + taxonomy, project, chr, start, ref, + alt, rs, false, false, false, false, + 1); if (backpropRs != null) { submittedVariantEntity.setBackPropagatedVariantAccession(backpropRs); } @@ -118,7 +116,7 @@ private SubmittedVariantEntity buildVariant(String chr, long start, String ref, @Test public void writeBasicFields() throws Exception { - File outputFolder = temporaryFolder.newFolder(); + File outputFolder = temporaryFolderUtil.newFolder(); String chromosome = "1"; long start = 1000; String reference = "C"; @@ -151,7 +149,7 @@ private long forEachVcfDataLine(File output, Consumer processVcfDataLi @Test public void writeRsId() throws Exception { - File outputFolder = temporaryFolder.newFolder(); + File outputFolder = temporaryFolderUtil.newFolder(); long rsId = 123L; File output = assertWriteVcf(outputFolder, buildVariant("1", 1000, "C", "A", rsId)); @@ -186,11 +184,11 @@ private void assertInfo(String infoColumn, String expectedSSHash, Long expectedR assertEquals("RS=rs" + expectedRsId, info); } else if (info.startsWith("BACKPROP_RS=")) { assertEquals("BACKPROP_RS=rs" + expectedBackpropRs, info); - } else if(info.startsWith("PROJECT=")) { + } else if (info.startsWith("PROJECT=")) { assertEquals("PROJECT=" + expectedProject, info); - } else if(info.startsWith("TAX=")) { + } else if (info.startsWith("TAX=")) { assertEquals("TAX=" + expectedTaxonomy, info); - } else if(info.startsWith("CREATED=")) { + } else if (info.startsWith("CREATED=")) { assertEquals("CREATED=" + expectedCreatedDate, info); } } @@ -198,7 +196,7 @@ private void assertInfo(String infoColumn, String expectedSSHash, Long expectedR @Test public void writeProjectAndTaxonomy() throws Exception { - File outputFolder = temporaryFolder.newFolder(); + File outputFolder = temporaryFolderUtil.newFolder(); String project2 = "project2"; int taxonomy2 = 1000; File output = assertWriteVcf(outputFolder, buildVariant("1", 1000, "C", "A", null, project2, taxonomy2)); @@ -212,7 +210,7 @@ public void writeProjectAndTaxonomy() throws Exception { @Test public void writeProjectAndTaxonomyAndRs() throws Exception { - File outputFolder = temporaryFolder.newFolder(); + File outputFolder = temporaryFolderUtil.newFolder(); String project2 = "project2"; int taxonomy2 = 1000; long rsId = 123L; @@ -228,16 +226,16 @@ public void writeProjectAndTaxonomyAndRs() throws Exception { @Test public void writeSeveralVariants() throws Exception { - File outputFolder = temporaryFolder.newFolder(); + File outputFolder = temporaryFolderUtil.newFolder(); String project2 = "project2"; int taxonomy2 = 1000; long rsId = 123L; long backpropRs = 456L; File output = assertWriteVcf(outputFolder, - buildVariant("1", 1000, "C", "A", rsId, backpropRs, project2, taxonomy2), - buildVariant("2", 1000, "C", "A", rsId, backpropRs, project2, taxonomy2), - buildVariant("3", 1000, "C", "A", rsId, backpropRs, project2, taxonomy2), - buildVariant("4", 1000, "C", "A", rsId, backpropRs, project2, taxonomy2)); + buildVariant("1", 1000, "C", "A", rsId, backpropRs, project2, taxonomy2), + buildVariant("2", 1000, "C", "A", rsId, backpropRs, project2, taxonomy2), + buildVariant("3", 1000, "C", "A", rsId, backpropRs, project2, taxonomy2), + buildVariant("4", 1000, "C", "A", rsId, backpropRs, project2, taxonomy2)); final Long[] expectedChr = {1L}; long variantCount = forEachVcfDataLine(output, (String[] columns) -> { @@ -251,10 +249,10 @@ public void writeSeveralVariants() throws Exception { @Test public void writeProjectWithSpecialCharacters() throws Exception { - File outputFolder = temporaryFolder.newFolder(); + File outputFolder = temporaryFolderUtil.newFolder(); String specialProject = "a%weird=project;with,special characters"; File output = assertWriteVcf(outputFolder, buildVariant("1", 1000, "C", "A", null, specialProject, - TAXONOMY_ACCESSION)); + TAXONOMY_ACCESSION)); long variantCount = forEachVcfDataLine(output, (String[] columns) -> { assertEquals(COLUMNS_IN_VCF_WITHOUT_SAMPLES, columns.length); diff --git a/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/batch/processors/ContextNucleotideAdditionProcessorTest.java b/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/batch/processors/ContextNucleotideAdditionProcessorTest.java index 06a764df5..96623a5a7 100644 --- a/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/batch/processors/ContextNucleotideAdditionProcessorTest.java +++ b/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/batch/processors/ContextNucleotideAdditionProcessorTest.java @@ -17,10 +17,9 @@ */ package uk.ac.ebi.eva.remapping.source.batch.processors; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; - +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; import uk.ac.ebi.eva.accession.core.batch.io.FastaSynonymSequenceReader; import uk.ac.ebi.eva.accession.core.contig.ContigMapping; import uk.ac.ebi.eva.accession.core.contig.ContigSynonyms; @@ -31,9 +30,10 @@ import java.nio.file.Paths; import java.util.Arrays; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThrows; -import static org.junit.Assert.fail; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; public class ContextNucleotideAdditionProcessorTest { @@ -51,18 +51,18 @@ public class ContextNucleotideAdditionProcessorTest { private static ContextNucleotideAdditionProcessor contextNucleotideAdditionProcessor; - @BeforeClass + @BeforeAll public static void setUpClass() throws Exception { Path fastaPath = Paths.get("../eva-accession-core/src/test/resources/input-files/fasta/Gallus_gallus-5.0.test.fa"); ContigMapping contigMapping = new ContigMapping( Arrays.asList(new ContigSynonyms(CONTIG, "", "", "", "", "", true), - new ContigSynonyms(SCAFFOLD, "", "", SCAFFOLD_GENBANK_IN_FASTA, "", "", true), - new ContigSynonyms(MISSING_IN_FASTA, "", "", "", "", "", true))); + new ContigSynonyms(SCAFFOLD, "", "", SCAFFOLD_GENBANK_IN_FASTA, "", "", true), + new ContigSynonyms(MISSING_IN_FASTA, "", "", "", "", "", true))); fastaSynonymSequenceReader = new FastaSynonymSequenceReader(contigMapping, fastaPath); contextNucleotideAdditionProcessor = new ContextNucleotideAdditionProcessor(fastaSynonymSequenceReader); } - @AfterClass + @AfterAll public static void tearDownClass() throws Exception { fastaSynonymSequenceReader.close(); } @@ -97,8 +97,9 @@ public void testNonEmptyAlleles() throws Exception { private SubmittedVariantEntity createVariant(String chromosome, long start, String reference, String alternate) { return new SubmittedVariantEntity(1L, "hash", "GCA_x", 9999, "project", chromosome, start, reference, alternate, - 100L, false, false, false, false, 1); + 100L, false, false, false, false, 1); } + @Test public void testINDELStartPos1() throws Exception { SubmittedVariantEntity variant = createVariant(CONTIG, 1, "", "A"); @@ -179,15 +180,18 @@ public void addContextBaseUsingSynonymContig() throws Exception { assertEquals("TA", processedVariant.getAlternateAllele()); } - @Test(expected = IllegalArgumentException.class) - public void contigNotFound() throws Exception { + @Test + public void contigNotFound() { SubmittedVariantEntity variant1 = createVariant(MISSING_IN_FASTA, 10, "", "A"); - try { - contextNucleotideAdditionProcessor.process(variant1); - } catch (PositionOutsideOfContigException wrongException) { - fail("The exception (" + wrongException.getClass().getSimpleName() - + ") is wrong because the variant doesn't have a position outside of chromosome. The correct " - + "exception should be that the contig is not present in the fasta"); + + Exception exception = assertThrows(Exception.class, () -> contextNucleotideAdditionProcessor.process(variant1)); + + if (exception instanceof PositionOutsideOfContigException) { + fail("The exception (" + exception.getClass().getSimpleName() + + ") is wrong because the variant doesn't have a position outside of chromosome. The correct " + + "exception should be that the contig is not present in the fasta"); } + + assertTrue(exception instanceof IllegalArgumentException); } } diff --git a/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/batch/processors/ExcludeAllelesMismatchVariantsProcessorTest.java b/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/batch/processors/ExcludeAllelesMismatchVariantsProcessorTest.java index 7ffd57ff1..9f23962b0 100644 --- a/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/batch/processors/ExcludeAllelesMismatchVariantsProcessorTest.java +++ b/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/batch/processors/ExcludeAllelesMismatchVariantsProcessorTest.java @@ -15,8 +15,8 @@ */ package uk.ac.ebi.eva.remapping.source.batch.processors; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import uk.ac.ebi.ampt2d.commons.accession.hashing.SHA1HashingFunction; import uk.ac.ebi.eva.accession.core.model.ISubmittedVariant; @@ -26,8 +26,8 @@ import java.util.function.Function; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; public class ExcludeAllelesMismatchVariantsProcessorTest { @@ -36,7 +36,7 @@ public class ExcludeAllelesMismatchVariantsProcessorTest { private ExcludeAllelesMismatchVariantsProcessor processor; - @Before + @BeforeEach public void setUp() { processor = new ExcludeAllelesMismatchVariantsProcessor(); } @@ -59,6 +59,6 @@ private SubmittedVariantEntity createSubmittedVariantEntity(Boolean allelesMatch SubmittedVariant submittedVariant = new SubmittedVariant("asm", 1000, "project", "contig", 100, "A", "T", null); String hash = hashingFunction.apply(submittedVariant); return new SubmittedVariantEntity(5000000000L, hash, "asm", 1000, "project", "contig", 100, "A", "T", null, - false, false, allelesMatch, false, 1, null); + false, false, allelesMatch, false, 1, null); } } diff --git a/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/batch/processors/ExcludeMultimapVariantsProcessorTest.java b/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/batch/processors/ExcludeMultimapVariantsProcessorTest.java index 579c4a41e..4bfa19530 100644 --- a/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/batch/processors/ExcludeMultimapVariantsProcessorTest.java +++ b/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/batch/processors/ExcludeMultimapVariantsProcessorTest.java @@ -15,8 +15,8 @@ */ package uk.ac.ebi.eva.remapping.source.batch.processors; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import uk.ac.ebi.ampt2d.commons.accession.hashing.SHA1HashingFunction; import uk.ac.ebi.eva.accession.core.model.ISubmittedVariant; @@ -26,8 +26,8 @@ import java.util.function.Function; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; public class ExcludeMultimapVariantsProcessorTest { @@ -35,7 +35,7 @@ public class ExcludeMultimapVariantsProcessorTest { private ExcludeMultimapVariantsProcessor processor; - @Before + @BeforeEach public void setUp() { processor = new ExcludeMultimapVariantsProcessor(); submittedHashingFunction = new SubmittedVariantSummaryFunction().andThen(new SHA1HashingFunction()); @@ -66,9 +66,9 @@ private SubmittedVariantEntity createSubmittedVariantEntity(Integer mapWeight) { SubmittedVariant submittedVariant = new SubmittedVariant("asm", 1000, "project", "contig", 100, "A", "T", null); String hash = submittedHashingFunction.apply(submittedVariant); SubmittedVariantEntity submittedVariantEntity = new SubmittedVariantEntity(5000000000L, hash, "asm", 1000, - "project", "contig", 100, "A", "T", - null, false, false, false, false, 1, - mapWeight); + "project", "contig", 100, "A", "T", + null, false, false, false, false, 1, + mapWeight); return submittedVariantEntity; } } \ No newline at end of file diff --git a/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/batch/processors/VariantToVariantContextProcessorTest.java b/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/batch/processors/VariantToVariantContextProcessorTest.java index b9d2d4095..413936119 100644 --- a/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/batch/processors/VariantToVariantContextProcessorTest.java +++ b/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/batch/processors/VariantToVariantContextProcessorTest.java @@ -18,17 +18,16 @@ import htsjdk.variant.variantcontext.Allele; import htsjdk.variant.variantcontext.VariantContext; -import org.junit.Before; -import org.junit.Test; - +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; import java.time.LocalDateTime; import java.util.Collections; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThrows; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; public class VariantToVariantContextProcessorTest { @@ -50,7 +49,7 @@ public class VariantToVariantContextProcessorTest { private SubmittedVariantToVariantContextProcessor variantConverter; - @Before + @BeforeEach public void setUp() { variantConverter = new SubmittedVariantToVariantContextProcessor(); } @@ -65,8 +64,8 @@ public void SNV() { private SubmittedVariantEntity buildVariant(String chr1, int start, String reference, String alternate, String project, int taxonomy) { SubmittedVariantEntity variant = new SubmittedVariantEntity(SS_ID, "hash", "assembly", taxonomy, project, chr1, - start, reference, alternate, RS_ID, - false, false, false, false, 1); + start, reference, alternate, RS_ID, + false, false, false, false, 1); variant.setCreatedDate(LocalDateTime.parse(CREATED_DATE)); return variant; } @@ -78,7 +77,7 @@ private void assertVariantContext(VariantContext variantContext, Long expectedId assertEquals(expectedStart, variantContext.getStart()); assertEquals(Allele.create(expectedReference, true), variantContext.getReference()); assertEquals(Collections.singletonList(Allele.create(expectedAlternate, false)), - variantContext.getAlternateAlleles()); + variantContext.getAlternateAlleles()); assertEquals("ss" + expectedId, variantContext.getID()); assertTrue(variantContext.getFilters().isEmpty()); assertEquals(5, variantContext.getCommonInfo().getAttributes().size()); @@ -88,7 +87,7 @@ private void assertVariantContext(VariantContext variantContext, Long expectedId assertTrue(variantContext.getCommonInfo().hasAttribute(TAXONOMY_ID_KEY)); assertEquals(expectedTaxonomy, - Integer.parseInt(variantContext.getCommonInfo().getAttribute(TAXONOMY_ID_KEY).toString())); + Integer.parseInt(variantContext.getCommonInfo().getAttribute(TAXONOMY_ID_KEY).toString())); assertEquals(0, variantContext.getSampleNames().size()); } diff --git a/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/configuration/batch/jobs/ExportSubmittedVariantsJobConfigurationTest.java b/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/configuration/batch/jobs/ExportSubmittedVariantsJobConfigurationTest.java index 6d2c0853b..6544bb0c9 100644 --- a/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/configuration/batch/jobs/ExportSubmittedVariantsJobConfigurationTest.java +++ b/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/configuration/batch/jobs/ExportSubmittedVariantsJobConfigurationTest.java @@ -15,31 +15,30 @@ */ package uk.ac.ebi.eva.remapping.source.configuration.batch.jobs; -import com.lordofthejars.nosqlunit.annotation.UsingDataSet; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.batch.core.BatchStatus; import org.springframework.batch.core.JobExecution; import org.springframework.batch.core.StepExecution; import org.springframework.batch.test.JobLauncherTestUtils; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.ApplicationContext; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.core.io.ResourceLoader; +import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; - -import uk.ac.ebi.eva.accession.core.test.rule.FixSpringMongoDbRule; +import org.springframework.test.context.junit.jupiter.SpringExtension; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; +import uk.ac.ebi.eva.accession.core.utils.MongoTestDataLoader; +import uk.ac.ebi.eva.commons.core.utils.FileUtils; +import uk.ac.ebi.eva.remapping.source.configuration.BeanNames; import uk.ac.ebi.eva.remapping.source.parameters.InputParameters; import uk.ac.ebi.eva.remapping.source.parameters.ReportPathResolver; +import uk.ac.ebi.eva.remapping.source.test.configuration.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.remapping.source.test.configuration.BatchTestConfiguration; import uk.ac.ebi.eva.remapping.source.test.configuration.MongoTestConfiguration; -import uk.ac.ebi.eva.commons.core.utils.FileUtils; -import uk.ac.ebi.eva.remapping.source.configuration.BeanNames; import java.io.FileInputStream; import java.io.FileNotFoundException; @@ -48,55 +47,56 @@ import java.util.List; import java.util.stream.Collectors; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static uk.ac.ebi.eva.remapping.source.test.configuration.BatchTestConfiguration.JOB_EXPORT_SUBMITTED_VARIANTS_JOB; -@RunWith(SpringRunner.class) -@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class}) -@UsingDataSet(locations = { - "/test-data/dbsnpSubmittedVariantEntity.json", - "/test-data/submittedVariantEntity.json"}) +@ExtendWith(SpringExtension.class) @TestPropertySource("classpath:with-projects.properties") -public class ExportSubmittedVariantsJobConfigurationTest { - - private static final String TEST_DB = "test-db"; - +@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class, + BatchJobRepositoryTestConfiguration.class}) +public class ExportSubmittedVariantsJobConfigurationTest extends MongoTestContainerHelper { private static final long EXPECTED_LINES_DBSNP = 3; private static final long EXPECTED_LINES_EVA = 1; @Autowired + @Qualifier(JOB_EXPORT_SUBMITTED_VARIANTS_JOB) private JobLauncherTestUtils jobLauncherTestUtils; @Autowired private InputParameters inputParameters; - //Required by nosql-unit @Autowired - private ApplicationContext applicationContext; + private MongoTemplate mongoTemplate; - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName(TEST_DB).build()); + @Autowired + private ResourceLoader resourceLoader; - @Before + @BeforeEach public void setUp() throws Exception { + mongoTemplate.getDb().drop(); deleteOutputFiles(); + + MongoTestDataLoader mongoTestDataLoader = new MongoTestDataLoader(mongoTemplate, resourceLoader); + mongoTestDataLoader.load("/test-data/dbsnpSubmittedVariantEntity.json"); + mongoTestDataLoader.load("/test-data/submittedVariantEntity.json"); } - @After + @AfterEach public void tearDown() throws Exception { + mongoTemplate.getDb().drop(); deleteOutputFiles(); } private void deleteOutputFiles() { ReportPathResolver.getDbsnpReportPath(inputParameters.getOutputFolder(), - inputParameters.getAssemblyAccession(), - inputParameters.getTaxonomy()) - .toFile().delete(); + inputParameters.getAssemblyAccession(), + inputParameters.getTaxonomy()) + .toFile().delete(); ReportPathResolver.getEvaReportPath(inputParameters.getOutputFolder(), - inputParameters.getAssemblyAccession(), - inputParameters.getTaxonomy()) - .toFile().delete(); + inputParameters.getAssemblyAccession(), + inputParameters.getTaxonomy()) + .toFile().delete(); } @Test @@ -104,7 +104,7 @@ public void basicJobCompletion() throws Exception { JobExecution jobExecution = jobLauncherTestUtils.launchJob(); List expectedSteps = Arrays.asList(BeanNames.EXPORT_EVA_SUBMITTED_VARIANTS_STEP, - BeanNames.EXPORT_DBSNP_SUBMITTED_VARIANTS_STEP); + BeanNames.EXPORT_DBSNP_SUBMITTED_VARIANTS_STEP); assertStepsExecuted(expectedSteps, jobExecution); assertEquals(BatchStatus.COMPLETED, jobExecution.getStatus()); @@ -122,16 +122,16 @@ public void variantsWritten() throws Exception { private FileInputStream getExportedEva() throws FileNotFoundException { return new FileInputStream(ReportPathResolver.getEvaReportPath(inputParameters.getOutputFolder(), - inputParameters.getAssemblyAccession(), - inputParameters.getTaxonomy()) - .toFile()); + inputParameters.getAssemblyAccession(), + inputParameters.getTaxonomy()) + .toFile()); } private FileInputStream getExportedDbsnp() throws FileNotFoundException { return new FileInputStream(ReportPathResolver.getDbsnpReportPath(inputParameters.getOutputFolder(), - inputParameters.getAssemblyAccession(), - inputParameters.getTaxonomy()) - .toFile()); + inputParameters.getAssemblyAccession(), + inputParameters.getTaxonomy()) + .toFile()); } private void assertStepsExecuted(List expectedSteps, JobExecution jobExecution) { diff --git a/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/runner/AccessionRemappingJobLauncherCommandLineRunnerTest.java b/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/runner/AccessionRemappingJobLauncherCommandLineRunnerTest.java index 8c61d6069..358595a01 100644 --- a/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/runner/AccessionRemappingJobLauncherCommandLineRunnerTest.java +++ b/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/runner/AccessionRemappingJobLauncherCommandLineRunnerTest.java @@ -15,72 +15,69 @@ */ package uk.ac.ebi.eva.remapping.source.runner; -import com.lordofthejars.nosqlunit.annotation.UsingDataSet; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.batch.core.JobExecutionException; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.ApplicationContext; +import org.springframework.core.io.ResourceLoader; +import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; - -import uk.ac.ebi.eva.accession.core.test.rule.FixSpringMongoDbRule; +import org.springframework.test.context.junit.jupiter.SpringExtension; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; +import uk.ac.ebi.eva.accession.core.utils.MongoTestDataLoader; import uk.ac.ebi.eva.remapping.source.parameters.InputParameters; import uk.ac.ebi.eva.remapping.source.parameters.ReportPathResolver; +import uk.ac.ebi.eva.remapping.source.test.configuration.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.remapping.source.test.configuration.BatchTestConfiguration; import uk.ac.ebi.eva.remapping.source.test.configuration.MongoTestConfiguration; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; -@RunWith(SpringRunner.class) -@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class}) -@UsingDataSet(locations = { - "/test-data/dbsnpSubmittedVariantEntity.json", - "/test-data/submittedVariantEntity.json"}) +@ExtendWith(SpringExtension.class) +@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class, + BatchJobRepositoryTestConfiguration.class}) @TestPropertySource("classpath:application.properties") -public class AccessionRemappingJobLauncherCommandLineRunnerTest { - - private static final String TEST_DB = "test-db"; - +public class AccessionRemappingJobLauncherCommandLineRunnerTest extends MongoTestContainerHelper { @Autowired private InputParameters inputParameters; @Autowired - private AccessionRemappingJobLauncherCommandLineRunner runner; + private MongoTemplate mongoTemplate; - //Required by nosql-unit @Autowired - private ApplicationContext applicationContext; + private ResourceLoader resourceLoader; - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName(TEST_DB).build()); + @Autowired + private AccessionRemappingJobLauncherCommandLineRunner runner; - @Before + @BeforeEach public void setUp() { + mongoTemplate.getDb().drop(); deleteOutputFiles(); + + MongoTestDataLoader mongoTestDataLoader = new MongoTestDataLoader(mongoTemplate, resourceLoader); + mongoTestDataLoader.load("/test-data/dbsnpSubmittedVariantEntity.json"); + mongoTestDataLoader.load("/test-data/submittedVariantEntity.json"); } - @After + @AfterEach public void tearDown() { + mongoTemplate.getDb().drop(); deleteOutputFiles(); } private void deleteOutputFiles() { ReportPathResolver.getDbsnpReportPath(inputParameters.getOutputFolder(), - inputParameters.getAssemblyAccession(), - inputParameters.getTaxonomy()) - .toFile().delete(); + inputParameters.getAssemblyAccession(), + inputParameters.getTaxonomy()) + .toFile().delete(); ReportPathResolver.getEvaReportPath(inputParameters.getOutputFolder(), - inputParameters.getAssemblyAccession(), - inputParameters.getTaxonomy()) - .toFile().delete(); + inputParameters.getAssemblyAccession(), + inputParameters.getTaxonomy()) + .toFile().delete(); } @Test diff --git a/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/runner/AccessionRemappingJobLauncherCommandLineRunnerWithProjectsTest.java b/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/runner/AccessionRemappingJobLauncherCommandLineRunnerWithProjectsTest.java index f246f14df..37a40d69a 100644 --- a/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/runner/AccessionRemappingJobLauncherCommandLineRunnerWithProjectsTest.java +++ b/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/runner/AccessionRemappingJobLauncherCommandLineRunnerWithProjectsTest.java @@ -15,38 +15,32 @@ */ package uk.ac.ebi.eva.remapping.source.runner; -import com.lordofthejars.nosqlunit.annotation.UsingDataSet; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.batch.core.JobExecutionException; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.ApplicationContext; +import org.springframework.core.io.ResourceLoader; +import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; - -import uk.ac.ebi.eva.accession.core.test.rule.FixSpringMongoDbRule; +import org.springframework.test.context.junit.jupiter.SpringExtension; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; +import uk.ac.ebi.eva.accession.core.utils.MongoTestDataLoader; import uk.ac.ebi.eva.remapping.source.parameters.InputParameters; import uk.ac.ebi.eva.remapping.source.parameters.ReportPathResolver; +import uk.ac.ebi.eva.remapping.source.test.configuration.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.remapping.source.test.configuration.BatchTestConfiguration; import uk.ac.ebi.eva.remapping.source.test.configuration.MongoTestConfiguration; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; -@RunWith(SpringRunner.class) -@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class}) -@UsingDataSet(locations = { - "/test-data/dbsnpSubmittedVariantEntity.json", - "/test-data/submittedVariantEntity.json"}) +@ExtendWith(SpringExtension.class) +@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class, + BatchJobRepositoryTestConfiguration.class}) @TestPropertySource("classpath:with-projects.properties") -public class AccessionRemappingJobLauncherCommandLineRunnerWithProjectsTest { - - private static final String TEST_DB = "test-db"; +public class AccessionRemappingJobLauncherCommandLineRunnerWithProjectsTest extends MongoTestContainerHelper { @Autowired private InputParameters inputParameters; @@ -54,33 +48,37 @@ public class AccessionRemappingJobLauncherCommandLineRunnerWithProjectsTest { @Autowired private AccessionRemappingJobLauncherCommandLineRunner runner; - //Required by nosql-unit @Autowired - private ApplicationContext applicationContext; + private MongoTemplate mongoTemplate; - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName(TEST_DB).build()); + @Autowired + private ResourceLoader resourceLoader; - @Before + @BeforeEach public void setUp() { + mongoTemplate.getDb().drop(); deleteOutputFiles(); + + MongoTestDataLoader mongoTestDataLoader = new MongoTestDataLoader(mongoTemplate, resourceLoader); + mongoTestDataLoader.load("/test-data/dbsnpSubmittedVariantEntity.json"); + mongoTestDataLoader.load("/test-data/submittedVariantEntity.json"); } - @After + @AfterEach public void tearDown() { deleteOutputFiles(); + mongoTemplate.getDb().drop(); } private void deleteOutputFiles() { ReportPathResolver.getDbsnpReportPath(inputParameters.getOutputFolder(), - inputParameters.getAssemblyAccession(), - inputParameters.getTaxonomy()) - .toFile().delete(); + inputParameters.getAssemblyAccession(), + inputParameters.getTaxonomy()) + .toFile().delete(); ReportPathResolver.getEvaReportPath(inputParameters.getOutputFolder(), - inputParameters.getAssemblyAccession(), - inputParameters.getTaxonomy()) - .toFile().delete(); + inputParameters.getAssemblyAccession(), + inputParameters.getTaxonomy()) + .toFile().delete(); } @Test diff --git a/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/test/configuration/BatchJobRepositoryTestConfiguration.java b/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/test/configuration/BatchJobRepositoryTestConfiguration.java new file mode 100644 index 000000000..57c837cc4 --- /dev/null +++ b/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/test/configuration/BatchJobRepositoryTestConfiguration.java @@ -0,0 +1,22 @@ +package uk.ac.ebi.eva.remapping.source.test.configuration; + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Primary; +import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseBuilder; +import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseType; + +import javax.sql.DataSource; + +@Configuration +public class BatchJobRepositoryTestConfiguration { + + @Bean + @Primary + public DataSource dataSource() { + return new EmbeddedDatabaseBuilder() + .setType(EmbeddedDatabaseType.HSQL) + .addScript("org/springframework/batch/core/schema-hsqldb.sql") + .build(); + } +} \ No newline at end of file diff --git a/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/test/configuration/BatchTestConfiguration.java b/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/test/configuration/BatchTestConfiguration.java index 8a79307b5..9a15f97c3 100644 --- a/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/test/configuration/BatchTestConfiguration.java +++ b/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/test/configuration/BatchTestConfiguration.java @@ -16,30 +16,42 @@ package uk.ac.ebi.eva.remapping.source.test.configuration; +import org.springframework.batch.core.Job; +import org.springframework.batch.core.launch.JobLauncher; +import org.springframework.batch.core.repository.JobRepository; import org.springframework.batch.test.JobLauncherTestUtils; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Import; - import uk.ac.ebi.eva.accession.core.configuration.nonhuman.MongoConfiguration; +import uk.ac.ebi.eva.commons.batch.job.JobExecutionApplicationListener; import uk.ac.ebi.eva.remapping.source.configuration.InputParametersConfiguration; import uk.ac.ebi.eva.remapping.source.configuration.batch.jobs.ExportSubmittedVariantsJobConfiguration; import uk.ac.ebi.eva.remapping.source.configuration.batch.policies.PoliciesConfiguration; import uk.ac.ebi.eva.remapping.source.runner.AccessionRemappingJobLauncherCommandLineRunner; -import uk.ac.ebi.eva.commons.batch.job.JobExecutionApplicationListener; + +import static uk.ac.ebi.eva.remapping.source.configuration.BeanNames.EXPORT_SUBMITTED_VARIANTS_JOB; @EnableAutoConfiguration @Import({MongoConfiguration.class, - InputParametersConfiguration.class, - ExportSubmittedVariantsJobConfiguration.class, - PoliciesConfiguration.class, - AccessionRemappingJobLauncherCommandLineRunner.class + InputParametersConfiguration.class, + ExportSubmittedVariantsJobConfiguration.class, + PoliciesConfiguration.class, + AccessionRemappingJobLauncherCommandLineRunner.class }) public class BatchTestConfiguration { - @Bean - public JobLauncherTestUtils jobLauncherTestUtils() { - return new JobLauncherTestUtils(); + public static final String JOB_EXPORT_SUBMITTED_VARIANTS_JOB = "JOB_EXPORT_SUBMITTED_VARIANTS_JOB"; + + @Bean(JOB_EXPORT_SUBMITTED_VARIANTS_JOB) + public JobLauncherTestUtils jobLauncherTestUtilsFromMongo(JobLauncher jobLauncher, JobRepository jobRepository, + @Qualifier(EXPORT_SUBMITTED_VARIANTS_JOB) Job job) { + JobLauncherTestUtils utils = new JobLauncherTestUtils(); + utils.setJobLauncher(jobLauncher); + utils.setJobRepository(jobRepository); + utils.setJob(job); + return utils; } @Bean diff --git a/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/test/configuration/MongoTestConfiguration.java b/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/test/configuration/MongoTestConfiguration.java index f8eed1fa0..1005a6f6a 100644 --- a/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/test/configuration/MongoTestConfiguration.java +++ b/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/test/configuration/MongoTestConfiguration.java @@ -17,12 +17,30 @@ */ package uk.ac.ebi.eva.remapping.source.test.configuration; +import com.mongodb.ReadPreference; +import com.mongodb.WriteConcern; +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoClients; +import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.domain.EntityScan; import org.springframework.boot.test.autoconfigure.data.mongo.AutoConfigureDataMongo; +import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Primary; +import org.springframework.data.mongodb.MongoDatabaseFactory; import org.springframework.data.mongodb.config.EnableMongoAuditing; +import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.data.mongodb.core.SimpleMongoClientDatabaseFactory; +import org.springframework.data.mongodb.core.WriteResultChecking; +import org.springframework.data.mongodb.core.convert.DefaultDbRefResolver; +import org.springframework.data.mongodb.core.convert.DefaultMongoTypeMapper; +import org.springframework.data.mongodb.core.convert.MappingMongoConverter; +import org.springframework.data.mongodb.core.convert.MongoCustomConversions; +import org.springframework.data.mongodb.core.mapping.MongoMappingContext; import org.springframework.data.mongodb.repository.config.EnableMongoRepositories; +import java.util.Collections; + @Configuration @EntityScan(basePackages = {"uk.ac.ebi.eva.accession.core.persistence"}) @EnableMongoRepositories(basePackages = { @@ -32,4 +50,56 @@ @AutoConfigureDataMongo public class MongoTestConfiguration { + @Value("${spring.data.mongodb.uri}") + private String mongoUri; + + @Value("${spring.data.mongodb.database}") + private String database; + + @Value("${mongodb.read-preference:primary}") + private String readPreference; + + @Bean + @Primary + public MongoClient mongoClient() { + return MongoClients.create(mongoUri); + } + + @Bean("primaryFactory") + @Primary + public MongoDatabaseFactory mongoDbFactory() { + return new SimpleMongoClientDatabaseFactory(mongoClient(), database); + } + + @Bean + @Primary + public MongoCustomConversions mongoCustomConversions() { + return new MongoCustomConversions(Collections.emptyList()); + } + + @Bean + @Primary + public MongoMappingContext mongoMappingContext(MongoCustomConversions conversions) { + MongoMappingContext context = new MongoMappingContext(); + context.setSimpleTypeHolder(conversions.getSimpleTypeHolder()); + context.setAutoIndexCreation(false); + return context; + } + + @Bean + @Primary + public MongoTemplate mongoTemplate(MongoMappingContext mongoMappingContext, + MongoCustomConversions conversions) { + MappingMongoConverter converter = new MappingMongoConverter( + new DefaultDbRefResolver(mongoDbFactory()), mongoMappingContext); + converter.setCustomConversions(conversions); + converter.setTypeMapper(new DefaultMongoTypeMapper(null)); + converter.afterPropertiesSet(); + MongoTemplate mongoTemplate = new MongoTemplate(mongoDbFactory(), converter); + mongoTemplate.setReadPreference(ReadPreference.valueOf(readPreference)); + mongoTemplate.setWriteConcern(WriteConcern.MAJORITY); + mongoTemplate.setWriteResultChecking(WriteResultChecking.EXCEPTION); + return mongoTemplate; + } + } diff --git a/eva-remapping-get-source/src/test/resources/application.properties b/eva-remapping-get-source/src/test/resources/application.properties index d8833dadf..941cf32a5 100644 --- a/eva-remapping-get-source/src/test/resources/application.properties +++ b/eva-remapping-get-source/src/test/resources/application.properties @@ -16,7 +16,6 @@ parameters.outputFolder=/tmp/ parameters.forceRestart=false parameters.chunkSize=1000 -spring.data.mongodb.uri=mongodb://|eva.mongo.host.test|:27017 spring.data.mongodb.database=test-db mongodb.read-preference=primary diff --git a/eva-remapping-get-source/src/test/resources/with-projects.properties b/eva-remapping-get-source/src/test/resources/with-projects.properties index fa9fd1511..d0c6bb68d 100644 --- a/eva-remapping-get-source/src/test/resources/with-projects.properties +++ b/eva-remapping-get-source/src/test/resources/with-projects.properties @@ -18,7 +18,6 @@ parameters.outputFolder=/tmp/ parameters.forceRestart=false parameters.chunkSize=1000 -spring.data.mongodb.uri=mongodb://|eva.mongo.host.test|:27017 spring.data.mongodb.database=test-db mongodb.read-preference=primary diff --git a/eva-remapping-ingest/pom.xml b/eva-remapping-ingest/pom.xml index 232fb9dab..d85d54e4b 100644 --- a/eva-remapping-ingest/pom.xml +++ b/eva-remapping-ingest/pom.xml @@ -2,12 +2,13 @@ + 4.0.0 + eva-accession uk.ac.ebi.eva - 0.6.57-SNAPSHOT + ${revision} - 4.0.0 eva-remapping-ingest jar @@ -27,6 +28,14 @@ xstream + + uk.ac.ebi.eva + eva-accession-core + ${revision} + test-jar + test + + org.springframework.batch spring-batch-test @@ -34,8 +43,18 @@ - com.lordofthejars - nosqlunit-mongodb + org.springframework.boot + spring-boot-testcontainers + test + + + org.testcontainers + mongodb + test + + + org.testcontainers + junit-jupiter test @@ -75,8 +94,4 @@ - - localhost - - \ No newline at end of file diff --git a/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/batch/io/RemappedSubmittedVariantsWriter.java b/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/batch/io/RemappedSubmittedVariantsWriter.java index e75f47636..a3232ee16 100644 --- a/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/batch/io/RemappedSubmittedVariantsWriter.java +++ b/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/batch/io/RemappedSubmittedVariantsWriter.java @@ -21,12 +21,12 @@ import org.apache.commons.lang3.tuple.ImmutableTriple; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.batch.item.Chunk; import org.springframework.batch.item.ItemWriter; -import org.springframework.dao.DuplicateKeyException; +import org.springframework.data.mongodb.BulkOperationException; import org.springframework.data.mongodb.core.BulkOperations; import org.springframework.data.mongodb.core.MongoTemplate; import uk.ac.ebi.ampt2d.commons.accession.core.models.EventType; - import uk.ac.ebi.eva.accession.core.exceptions.MongoBulkWriteExceptionUtils; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantInactiveEntity; @@ -75,9 +75,9 @@ public RemappedSubmittedVariantsWriter(MongoTemplate mongoTemplate, String assem } @Override - public void write(List submittedVariantsRemapped) { + public void write(Chunk submittedVariantsRemapped) { // We do this to avoid tedious generic type signatures in the method calls - List svesToInsert = new ArrayList<>(submittedVariantsRemapped); + List svesToInsert = new ArrayList<>(submittedVariantsRemapped.getItems()); // Resolve duplicate hashes and accessions before inserting; note that hash resolution must happen first. ImmutablePair, List> duplicateHashDiscards = @@ -86,26 +86,26 @@ public void write(List submittedVariantsRemapp resolveDuplicateAccessions(svesToInsert); List svesToDiscard = Stream.concat(duplicateHashDiscards.getLeft().stream(), - duplicateAccessionDiscards.getLeft().stream()) - .collect(Collectors.toList()); + duplicateAccessionDiscards.getLeft().stream()) + .collect(Collectors.toList()); List discardOperations = Stream.concat(duplicateHashDiscards.getRight().stream(), - duplicateAccessionDiscards.getRight().stream()) - .collect(Collectors.toList()); + duplicateAccessionDiscards.getRight().stream()) + .collect(Collectors.toList()); BulkOperations bulkOperations = mongoTemplate.bulkOps(BulkOperations.BulkMode.UNORDERED, - SubmittedVariantEntity.class, - collection); + SubmittedVariantEntity.class, + collection); // Deal with discards before inserts, to avoid DuplicateKeyExceptions if (discardOperations.size() > 0) { try { BulkOperations svoeBulkOperations = mongoTemplate.bulkOps(BulkOperations.BulkMode.UNORDERED, - SubmittedVariantOperationEntity.class, - operationCollection); + SubmittedVariantOperationEntity.class, + operationCollection); svoeBulkOperations.insert(discardOperations); BulkWriteResult bulkWriteResult = svoeBulkOperations.execute(); remappingIngestCounts.addRemappedVariantsDiscarded(bulkWriteResult.getInsertedCount()); - } catch (DuplicateKeyException exception) { + } catch (BulkOperationException exception) { // As we check for hash collisions in submitted variants, we should only get duplicate keys from // trying to insert an identical DISCARD operation, which shouldn't happen even when rerunning // (since identical SVEs are skipped). @@ -123,7 +123,7 @@ public void write(List submittedVariantsRemapp if (svesToDiscard.size() > 0) { bulkOperations.remove(query(where("_id").in( svesToDiscard.stream().map(SubmittedVariantEntity::getHashedMessage) - .collect(Collectors.toSet())))); + .collect(Collectors.toSet())))); bulkOperations.execute(); } @@ -138,17 +138,17 @@ private ImmutablePair, List svesToInsert) { Map> svesGroupedByHash = getDuplicateHashes(svesToInsert); return resolveDuplicates(svesToInsert, svesGroupedByHash, - "Submitted variant discarded due to duplicate hash"); + "Submitted variant discarded due to duplicate hash"); } private ImmutablePair, List> resolveDuplicateAccessions( List svesToInsert) { Map> svesGroupedByAccession = getDuplicateAccessions(svesToInsert); return resolveDuplicates(svesToInsert, svesGroupedByAccession, - "Submitted variant discarded due to duplicate SS IDs in assembly"); + "Submitted variant discarded due to duplicate SS IDs in assembly"); } - private ImmutablePair, List> resolveDuplicates( + private ImmutablePair, List> resolveDuplicates( List svesToInsert, Map> svesGroupedByKey, String discardMessage) { @@ -159,7 +159,7 @@ private ImmutablePair, List, LocalDateTime> createdDateMap = getAllCreatedDateFromSource(duplicateSves); - for (Map.Entry> keyAndSve: svesGroupedByKey.entrySet()) { + for (Map.Entry> keyAndSve : svesGroupedByKey.entrySet()) { List duplicates = keyAndSve.getValue(); // Ensure we only keep one out of the list of duplicates for this SS. SubmittedVariantEntity currentKept = duplicates.get(0); @@ -177,7 +177,7 @@ private ImmutablePair, ListImmutablePair, List> getDuplicateHashes(List svesToInsert) { List hashes = svesToInsert.stream() - .map(SubmittedVariantEntity::getHashedMessage) - .collect(Collectors.toList()); + .map(SubmittedVariantEntity::getHashedMessage) + .collect(Collectors.toList()); // Find duplicate hashes already in db List svesWithSameHash = mongoTemplate.find(query(where("_id").in(hashes)), - SubmittedVariantEntity.class, collection); + SubmittedVariantEntity.class, collection); // Get the subset of these that are actually identical (including accession) and remove them immediately from // svesToInsert. These are counted as skips rather than discards. Map> duplicateSve = filterForDuplicates( Stream.concat(svesToInsert.stream(), svesWithSameHash.stream()) - .collect(Collectors.groupingBy(sve -> sve.hashCode() + "_" + sve.getAccession()))); + .collect(Collectors.groupingBy(sve -> sve.hashCode() + "_" + sve.getAccession()))); for (List dups : duplicateSve.values()) { int numRemoved = removeAllFromSveList(svesToInsert, dups.get(0)); remappingIngestCounts.addRemappedVariantsSkipped(numRemoved); @@ -230,7 +230,7 @@ private Map> getDuplicateHashes(List> svesGroupedByHash = Stream.concat(svesToInsert.stream(), svesWithSameHash.stream()) - .collect(Collectors.groupingBy(SubmittedVariantEntity::getHashedMessage)); + .collect(Collectors.groupingBy(SubmittedVariantEntity::getHashedMessage)); return filterForDuplicates(svesGroupedByHash); } @@ -242,7 +242,7 @@ private int removeAllFromSveList(List sves, SubmittedVar } private boolean containsSveList(List sves, SubmittedVariantEntity sveToFind) { - for (SubmittedVariantEntity sve: sves) { + for (SubmittedVariantEntity sve : sves) { if (equalsSveAndAccession(sve, sveToFind)) { return true; } @@ -256,8 +256,8 @@ private boolean equalsSveAndAccession(SubmittedVariantEntity first, SubmittedVar private Map> getDuplicateAccessions(List svesToInsert) { List accessions = svesToInsert.stream() - .map(SubmittedVariantEntity::getAccession) - .collect(Collectors.toList()); + .map(SubmittedVariantEntity::getAccession) + .collect(Collectors.toList()); // Find duplicate SS already in the database List svesWithSameAccession = mongoTemplate.find( @@ -266,7 +266,7 @@ private Map> getDuplicateAccessions(List> svesGroupedBySs = Stream.concat(svesToInsert.stream(), svesWithSameAccession.stream()) - .collect(Collectors.groupingBy(SubmittedVariantEntity::getAccession)); + .collect(Collectors.groupingBy(SubmittedVariantEntity::getAccession)); return filterForDuplicates(svesGroupedBySs); } @@ -278,8 +278,8 @@ private Map, LocalDateTime> getAllCreatedD // duplicates in all cases. If there's a duplicate triple almost certainly the createdDates are the same. Map, LocalDateTime> targetToSourceCreatedDate = duplicateSves .stream().collect(Collectors.toMap(this::getKeyForCreatedDate, - SubmittedVariantEntity::getCreatedDate, - (cd1, cd2) -> Collections.min(Arrays.asList(cd1, cd2)))); + SubmittedVariantEntity::getCreatedDate, + (cd1, cd2) -> Collections.min(Arrays.asList(cd1, cd2)))); Map> svesBySourceAssembly = duplicateSves .stream().collect(Collectors.groupingBy( @@ -323,42 +323,42 @@ private LocalDateTime getCreatedDateFromSource(SubmittedVariantEntity targetSve, // If we can't find the source SVE, use the created date of the remapped SVE if (Objects.isNull(sourceSves) || sourceSves.isEmpty()) { logger.warn("No SS " + targetSve.getAccession() + " found in source assembly " - + targetSve.getRemappedFrom()); + + targetSve.getRemappedFrom()); return targetSve.getCreatedDate(); } // If we find multiple SS in the source, return the min created date if (sourceSves.size() > 1) { logger.warn("Duplicate SS " + targetSve.getAccession() + " found in source assembly " - + targetSve.getRemappedFrom()); + + targetSve.getRemappedFrom()); return Collections.min(sourceSves, Comparator.comparing(SubmittedVariantEntity::getCreatedDate)) - .getCreatedDate(); + .getCreatedDate(); } return sourceSves.get(0).getCreatedDate(); } private SubmittedVariantOperationEntity getDiscardOperationWithMessage(SubmittedVariantEntity sve, String message) { String svoeId = String.format("DISCARD_SS_%s_HASH_%s_SOURCE_%s", - sve.getAccession(), - sve.getHashedMessage(), - sve.getRemappedFrom()); + sve.getAccession(), + sve.getHashedMessage(), + sve.getRemappedFrom()); SubmittedVariantOperationEntity svoe = new SubmittedVariantOperationEntity(); svoe.fill(EventType.DISCARDED, - sve.getAccession(), - message, - Collections.singletonList(new SubmittedVariantInactiveEntity(sve))); + sve.getAccession(), + message, + Collections.singletonList(new SubmittedVariantInactiveEntity(sve))); svoe.setId(svoeId); return svoe; } - private Map> filterForDuplicates(Map> map) { + private Map> filterForDuplicates(Map> map) { return map.entrySet().stream() - .filter(candidateEntry -> candidateEntry.getValue().size() > 1) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + .filter(candidateEntry -> candidateEntry.getValue().size() > 1) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); } - private List flattenValues(Map> map) { + private List flattenValues(Map> map) { return map.values().stream() - .flatMap(List::stream) - .collect(Collectors.toList()); + .flatMap(List::stream) + .collect(Collectors.toList()); } } diff --git a/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/configuration/RunnerConfiguration.java b/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/configuration/RunnerConfiguration.java index f2e7e7111..1b4b4b90b 100644 --- a/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/configuration/RunnerConfiguration.java +++ b/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/configuration/RunnerConfiguration.java @@ -16,16 +16,10 @@ */ package uk.ac.ebi.eva.remapping.ingest.configuration; -import org.springframework.batch.core.configuration.annotation.BatchConfigurer; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; - -import uk.ac.ebi.eva.commons.batch.configuration.SpringBoot1CompatibilityConfiguration; import uk.ac.ebi.eva.commons.batch.job.JobExecutionApplicationListener; -import javax.persistence.EntityManagerFactory; -import javax.sql.DataSource; - @Configuration public class RunnerConfiguration { @@ -33,11 +27,4 @@ public class RunnerConfiguration { public JobExecutionApplicationListener jobExecutionApplicationListener() { return new JobExecutionApplicationListener(); } - - @Bean - public BatchConfigurer configurer(DataSource dataSource, EntityManagerFactory entityManagerFactory) - throws Exception { - return SpringBoot1CompatibilityConfiguration.getSpringBoot1CompatibleBatchConfigurer(dataSource, - entityManagerFactory); - } } diff --git a/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/io/VcfReaderConfiguration.java b/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/io/VcfReaderConfiguration.java index a725ed28c..d2fe6a889 100644 --- a/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/io/VcfReaderConfiguration.java +++ b/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/io/VcfReaderConfiguration.java @@ -20,15 +20,14 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Import; - import uk.ac.ebi.eva.commons.batch.io.AggregatedVcfLineMapper; import uk.ac.ebi.eva.commons.batch.io.UnwindingItemStreamReader; import uk.ac.ebi.eva.commons.batch.io.VcfReader; import uk.ac.ebi.eva.commons.core.models.Aggregation; import uk.ac.ebi.eva.commons.core.models.pipeline.Variant; +import uk.ac.ebi.eva.remapping.ingest.configuration.BeanNames; import uk.ac.ebi.eva.remapping.ingest.configuration.InputParametersConfiguration; import uk.ac.ebi.eva.remapping.ingest.parameters.InputParameters; -import uk.ac.ebi.eva.remapping.ingest.configuration.BeanNames; import java.io.File; import java.io.IOException; diff --git a/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/jobs/IngestRemappedVariantsFromVcfJobConfiguration.java b/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/jobs/IngestRemappedVariantsFromVcfJobConfiguration.java index 46684370f..d7cee50c2 100644 --- a/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/jobs/IngestRemappedVariantsFromVcfJobConfiguration.java +++ b/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/jobs/IngestRemappedVariantsFromVcfJobConfiguration.java @@ -18,8 +18,9 @@ import org.springframework.batch.core.Job; import org.springframework.batch.core.Step; import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; -import org.springframework.batch.core.configuration.annotation.JobBuilderFactory; +import org.springframework.batch.core.job.builder.JobBuilder; import org.springframework.batch.core.launch.support.RunIdIncrementer; +import org.springframework.batch.core.repository.JobRepository; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -36,12 +37,12 @@ public class IngestRemappedVariantsFromVcfJobConfiguration { public Job ingestRemappedVariantsFromVcfJob( @Qualifier(INGEST_REMAPPED_VARIANTS_FROM_VCF_STEP) Step ingestRemappedVariantsStep, @Qualifier(STORE_REMAPPING_METADATA_STEP) Step storeRemappingMetadataStep, - JobBuilderFactory jobBuilderFactory) { - return jobBuilderFactory.get(INGEST_REMAPPED_VARIANTS_FROM_VCF_JOB) - .incrementer(new RunIdIncrementer()) - .start(storeRemappingMetadataStep) - .next(ingestRemappedVariantsStep) - .build(); + JobRepository jobRepository) { + return new JobBuilder(INGEST_REMAPPED_VARIANTS_FROM_VCF_JOB, jobRepository) + .incrementer(new RunIdIncrementer()) + .start(storeRemappingMetadataStep) + .next(ingestRemappedVariantsStep) + .build(); } } diff --git a/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/steps/IngestRemappedFromVcfStepConfiguration.java b/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/steps/IngestRemappedFromVcfStepConfiguration.java index ba4973144..e123343ca 100644 --- a/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/steps/IngestRemappedFromVcfStepConfiguration.java +++ b/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/steps/IngestRemappedFromVcfStepConfiguration.java @@ -19,7 +19,8 @@ import org.springframework.batch.core.Step; import org.springframework.batch.core.StepExecutionListener; import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; -import org.springframework.batch.core.configuration.annotation.StepBuilderFactory; +import org.springframework.batch.core.repository.JobRepository; +import org.springframework.batch.core.step.builder.StepBuilder; import org.springframework.batch.core.step.tasklet.TaskletStep; import org.springframework.batch.item.ItemProcessor; import org.springframework.batch.item.ItemReader; @@ -28,7 +29,7 @@ import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; - +import org.springframework.transaction.PlatformTransactionManager; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; import uk.ac.ebi.eva.commons.core.models.IVariant; import uk.ac.ebi.eva.commons.core.models.pipeline.Variant; @@ -49,15 +50,15 @@ public Step ingestRemappedFromVcf( @Qualifier(COMPOSITE_VARIANT_PROCESSOR) ItemProcessor processor, @Qualifier(REMAPPED_SUBMITTED_VARIANTS_WRITER) ItemWriter submittedVariantWriter, @Qualifier(PROGRESS_LISTENER) StepExecutionListener progressListener, - StepBuilderFactory stepBuilderFactory, + JobRepository jobRepository, PlatformTransactionManager transactionManager, SimpleCompletionPolicy chunkSizeCompletionPolicy) { - TaskletStep step = stepBuilderFactory.get(BeanNames.INGEST_REMAPPED_VARIANTS_FROM_VCF_STEP) - .chunk(chunkSizeCompletionPolicy) - .reader(vcfReader) - .processor(processor) - .writer(submittedVariantWriter) - .listener(progressListener) - .build(); + TaskletStep step = new StepBuilder(BeanNames.INGEST_REMAPPED_VARIANTS_FROM_VCF_STEP, jobRepository) + .chunk(chunkSizeCompletionPolicy, transactionManager) + .reader(vcfReader) + .processor(processor) + .writer(submittedVariantWriter) + .listener(progressListener) + .build(); return step; } diff --git a/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/steps/StoreRemappingMetadataStepConfiguration.java b/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/steps/StoreRemappingMetadataStepConfiguration.java index d4b0a1a9b..6d2c6803b 100644 --- a/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/steps/StoreRemappingMetadataStepConfiguration.java +++ b/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/steps/StoreRemappingMetadataStepConfiguration.java @@ -18,11 +18,13 @@ import org.springframework.batch.core.Step; import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; -import org.springframework.batch.core.configuration.annotation.StepBuilderFactory; +import org.springframework.batch.core.repository.JobRepository; +import org.springframework.batch.core.step.builder.StepBuilder; import org.springframework.batch.core.step.tasklet.TaskletStep; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.transaction.PlatformTransactionManager; import uk.ac.ebi.eva.remapping.ingest.batch.tasklets.RemappingMetadata; import uk.ac.ebi.eva.remapping.ingest.batch.tasklets.StoreRemappingMetadataTasklet; @@ -38,20 +40,17 @@ public class StoreRemappingMetadataStepConfiguration { private final RemappingMetadata remappingMetadata; -// @Value("${build.version}") -// private String accessionVersion; - public StoreRemappingMetadataStepConfiguration(MongoTemplate mongoTemplate, RemappingMetadata remappingMetadata) { this.mongoTemplate = mongoTemplate; this.remappingMetadata = remappingMetadata; } @Bean(STORE_REMAPPING_METADATA_STEP) - public Step buildReportStep(StepBuilderFactory stepBuilderFactory) throws IOException { + public Step buildReportStep(JobRepository jobRepository, PlatformTransactionManager transactionManager) throws IOException { StoreRemappingMetadataTasklet tasklet = new StoreRemappingMetadataTasklet(mongoTemplate, remappingMetadata); - TaskletStep step = stepBuilderFactory.get(STORE_REMAPPING_METADATA_STEP) - .tasklet(tasklet) - .build(); + TaskletStep step = new StepBuilder(STORE_REMAPPING_METADATA_STEP, jobRepository) + .tasklet(tasklet, transactionManager) + .build(); return step; } } diff --git a/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/runner/IngestRemappedCommandLineRunner.java b/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/runner/IngestRemappedCommandLineRunner.java index c78936437..021002fd9 100644 --- a/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/runner/IngestRemappedCommandLineRunner.java +++ b/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/runner/IngestRemappedCommandLineRunner.java @@ -26,7 +26,6 @@ import org.springframework.batch.core.JobParametersInvalidException; import org.springframework.batch.core.explore.JobExplorer; import org.springframework.batch.core.launch.JobLauncher; -import org.springframework.batch.core.launch.JobParametersNotFoundException; import org.springframework.batch.core.repository.JobExecutionAlreadyRunningException; import org.springframework.batch.core.repository.JobInstanceAlreadyCompleteException; import org.springframework.batch.core.repository.JobRepository; @@ -34,11 +33,10 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.ExitCodeGenerator; -import org.springframework.boot.autoconfigure.batch.JobLauncherCommandLineRunner; +import org.springframework.boot.autoconfigure.batch.JobLauncherApplicationRunner; import org.springframework.context.ApplicationEventPublisherAware; import org.springframework.stereotype.Component; import org.springframework.util.PatternMatchUtils; - import uk.ac.ebi.eva.accession.core.runner.CommandLineRunnerUtils; import uk.ac.ebi.eva.commons.batch.exception.NoJobToExecuteException; import uk.ac.ebi.eva.commons.batch.exception.NoParametersHaveBeenPassedException; @@ -51,7 +49,7 @@ import java.util.Collection; @Component -public class IngestRemappedCommandLineRunner extends JobLauncherCommandLineRunner implements +public class IngestRemappedCommandLineRunner extends JobLauncherApplicationRunner implements ApplicationEventPublisherAware, ExitCodeGenerator { private static final Logger logger = LoggerFactory.getLogger(IngestRemappedCommandLineRunner.class); @@ -92,9 +90,9 @@ public void setJobs(Collection jobs) { } @Override - public void setJobNames(String jobName) { + public void setJobName(String jobName) { this.jobName = jobName; - super.setJobNames(jobName); + super.setJobName(jobName); } @Override @@ -118,16 +116,15 @@ public void run(String... args) throws JobExecutionException { JobStatusManager.checkIfPropertiesHaveBeenProvided(jobParameters); if (inputParameters.isForceRestart()) { JobExecution previousJobExecution = CommandLineRunnerUtils.getLastJobExecution(jobName, jobExplorer, - jobParameters); + jobParameters); CommandLineRunnerUtils.markPreviousJobAsFailed(jobName, jobRepository, previousJobExecution != null ? previousJobExecution.getJobParameters() : jobParameters); - } - else { + } else { jobParameters = CommandLineRunnerUtils.addRunIDToJobParameters(jobName, jobExplorer, jobParameters); } launchJob(jobParameters); } catch (NoJobToExecuteException | NoParametersHaveBeenPassedException | NoPreviousJobExecutionException - | UnknownJobException | JobParametersInvalidException | JobExecutionAlreadyRunningException e) { + | UnknownJobException | JobParametersInvalidException | JobExecutionAlreadyRunningException e) { logger.error(e.getMessage()); logger.debug("Error trace", e); abnormalExit = true; @@ -149,8 +146,7 @@ private void launchJob(JobParameters jobParameters) throws JobExecutionException @Override protected void execute(Job job, JobParameters jobParameters) throws JobExecutionAlreadyRunningException, - JobRestartException, JobInstanceAlreadyCompleteException, JobParametersInvalidException, - JobParametersNotFoundException { + JobRestartException, JobInstanceAlreadyCompleteException, JobParametersInvalidException { logger.info("Running job '" + jobName + "' with parameters: " + jobParameters); super.execute(job, jobParameters); } diff --git a/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/batch/io/RemappedSubmittedVariantsWriterTest.java b/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/batch/io/RemappedSubmittedVariantsWriterTest.java index 08f5e0e8e..316e09f93 100644 --- a/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/batch/io/RemappedSubmittedVariantsWriterTest.java +++ b/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/batch/io/RemappedSubmittedVariantsWriterTest.java @@ -16,30 +16,28 @@ */ package uk.ac.ebi.eva.remapping.ingest.batch.io; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; -import com.mongodb.MongoClient; -import org.junit.After; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.batch.item.Chunk; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.context.ApplicationContext; +import org.springframework.core.io.ResourceLoader; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.ampt2d.commons.accession.hashing.SHA1HashingFunction; - import uk.ac.ebi.eva.accession.core.model.ISubmittedVariant; import uk.ac.ebi.eva.accession.core.model.SubmittedVariant; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantOperationEntity; import uk.ac.ebi.eva.accession.core.summary.SubmittedVariantSummaryFunction; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.remapping.ingest.batch.listeners.RemappingIngestCounts; import uk.ac.ebi.eva.remapping.ingest.test.configuration.BatchTestConfiguration; -import uk.ac.ebi.eva.remapping.ingest.test.rule.FixSpringMongoDbRule; +import uk.ac.ebi.eva.remapping.ingest.test.configuration.MongoTestConfiguration; import java.time.LocalDateTime; import java.util.Arrays; @@ -49,21 +47,18 @@ import java.util.Set; import java.util.function.Function; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import static uk.ac.ebi.eva.remapping.ingest.configuration.BeanNames.REMAPPED_SUBMITTED_VARIANTS_WRITER; -@RunWith(SpringRunner.class) -@ContextConfiguration(classes = {BatchTestConfiguration.class}) +@ExtendWith(SpringExtension.class) +@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class}) @TestPropertySource("classpath:ingest-remapped-variants.properties") -public class RemappedSubmittedVariantsWriterTest { - - private static final String TEST_DB = "test-ingest-remapping"; - +public class RemappedSubmittedVariantsWriterTest extends MongoTestContainerHelper { @Autowired - private MongoClient mongoClient; + private MongoTemplate mongoTemplate; @Autowired - private MongoTemplate mongoTemplate; + private ResourceLoader resourceLoader; @Autowired @Qualifier(REMAPPED_SUBMITTED_VARIANTS_WRITER) @@ -72,27 +67,25 @@ public class RemappedSubmittedVariantsWriterTest { private final Function hashingFunction = new SubmittedVariantSummaryFunction().andThen(new SHA1HashingFunction()); - //Required by nosql-unit @Autowired - private ApplicationContext applicationContext; + private RemappingIngestCounts remappingIngestCounts; - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName(TEST_DB).build()); + @BeforeEach + public void setUp() { + mongoTemplate.getDb().drop(); + } - @Autowired - private RemappingIngestCounts remappingIngestCounts; - @After + @AfterEach public void tearDown() { - mongoClient.dropDatabase(TEST_DB); + mongoTemplate.getDb().drop(); remappingIngestCounts.resetCounts(); } private SubmittedVariantEntity createSve(Long accession, long start, String ref, String alt, LocalDateTime createdDate, String remappedFrom) { SubmittedVariant model = new SubmittedVariant("GCA_000000001.2", 1000, "projectId_1", "CM000002.1", - start, ref, alt, 3000000002L); + start, ref, alt, 3000000002L); String hash = hashingFunction.apply(model); SubmittedVariantEntity sve = new SubmittedVariantEntity(accession, hash, model, 1); sve.setCreatedDate(createdDate); @@ -104,7 +97,7 @@ private SubmittedVariantEntity createSveInAssembly(String assembly, Long accession, long start, String ref, String alt, LocalDateTime createdDate, String remappedFrom) { SubmittedVariant model = new SubmittedVariant(assembly, 1000, "projectId_1", "CM000002.1", - start, ref, alt, 3000000002L); + start, ref, alt, 3000000002L); String hash = hashingFunction.apply(model); SubmittedVariantEntity sve = new SubmittedVariantEntity(accession, hash, model, 1); sve.setCreatedDate(createdDate); @@ -137,7 +130,7 @@ public void testDoesNotWriteNewDuplicateAccessions_discardFromInput() { List svesToWrite = Collections.singletonList( // SVE remapped from newer source SVE should be discarded createSve(5000000002L, 2100, "C", "T", LocalDateTime.now(), "GCA_000000001.5")); - writer.write(svesToWrite); + writer.write(new Chunk<>(svesToWrite)); assertRemappingIngestCounts(0, 0, 1); assertDatabaseCounts(3); } @@ -150,7 +143,7 @@ public void testDoesNotWriteNewDuplicateAccessions_discardFromDatabase() { List svesToWrite = Collections.singletonList( createSve(5000000004L, 1100, "C", "T", LocalDateTime.now(), null)); - writer.write(svesToWrite); + writer.write(new Chunk<>(svesToWrite)); assertRemappingIngestCounts(1, 0, 1); assertDatabaseCounts(1); } @@ -160,7 +153,7 @@ public void testDeduplicatesAccessionsFromInput() { List svesToWrite = Arrays.asList( createSve(5000000003L, 1100, "C", "T", LocalDateTime.now(), "GCA_000000001.1"), createSve(5000000003L, 1100, "A", "T", LocalDateTime.now(), "GCA_000000001.1")); - writer.write(svesToWrite); + writer.write(new Chunk<>(svesToWrite)); assertRemappingIngestCounts(1, 0, 1); assertDatabaseCounts(1); } @@ -172,7 +165,7 @@ public void testDiscardsDuplicateHashes() { List svesToWrite = Collections.singletonList( createSve(5000000005L, 1100, "C", "T", LocalDateTime.now(), "GCA_000000001.1")); - writer.write(svesToWrite); + writer.write(new Chunk<>(svesToWrite)); assertRemappingIngestCounts(0, 0, 1); assertDatabaseCounts(1); } @@ -187,7 +180,7 @@ public void testDeduplicatesHashesFromInput() { createSve(5000000003L, 1100, "C", "T", LocalDateTime.now(), "GCA_000000001.1"), // Duplicate hash but different accession => discard operation createSve(5000000004L, 1100, "C", "T", LocalDateTime.now(), "GCA_000000001.1")); - writer.write(svesToWrite); + writer.write(new Chunk<>(svesToWrite)); assertRemappingIngestCounts(0, 1, 1); assertDatabaseCounts(1); } @@ -201,7 +194,7 @@ public void testDuplicateHashAndAccessionButDifferentSVEs() { SubmittedVariantEntity duplicateSve = createSve(5000000003L, 1100, "C", "T", LocalDateTime.now(), "GCA_000000001.1"); duplicateSve.setBackPropagatedVariantAccession(12345L); List svesToWrite = Collections.singletonList(duplicateSve); - writer.write(svesToWrite); + writer.write(new Chunk<>(svesToWrite)); assertRemappingIngestCounts(0, 0, 1); assertDatabaseCounts(1); } @@ -212,7 +205,7 @@ public void testIdempotentWrites() { List svesToWrite = Collections.singletonList( createSve(5000000002L, 1100, "C", "T", LocalDateTime.now(), null)); - writer.write(svesToWrite); + writer.write(new Chunk<>(svesToWrite)); assertRemappingIngestCounts(1, 0, 1); Set sveAfterFirstWrite = new HashSet<>( @@ -221,7 +214,7 @@ public void testIdempotentWrites() { mongoTemplate.findAll(SubmittedVariantOperationEntity.class)); remappingIngestCounts.resetCounts(); - writer.write(svesToWrite); + writer.write(new Chunk<>(svesToWrite)); assertRemappingIngestCounts(0, 1, 0); Set sveAfterSecondWrite = new HashSet<>( @@ -249,7 +242,7 @@ public void testDuplicateAccessionInDb_discardFromDb() { List firstBatch = Collections.singletonList( createSve(5000000003L, 1400, "C", "T", now, "GCA_000000001.1") ); - writer.write(firstBatch); + writer.write(new Chunk<>(firstBatch)); assertRemappingIngestCounts(0, 0, 2); assertDatabaseCounts(1); @@ -257,7 +250,7 @@ public void testDuplicateAccessionInDb_discardFromDb() { createSve(5000000003L, 1400, "C", "T", now, "GCA_000000001.1"), createSve(5000000004L, 1400, "C", "T", now, "GCA_000000001.1") ); - writer.write(secondBatch); + writer.write(new Chunk<>(secondBatch)); assertRemappingIngestCounts(0, 1, 3); assertDatabaseCounts(1); } @@ -281,7 +274,7 @@ public void testDuplicateAccessionInDb_skipFromInput() { createSve(5000000003L, 1400, "C", "T", now, "GCA_000000001.1"), createSve(5000000004L, 1400, "C", "T", now, "GCA_000000001.1") ); - writer.write(svesToWrite); + writer.write(new Chunk<>(svesToWrite)); assertRemappingIngestCounts(1, 2, 0); assertDatabaseCounts(3); } diff --git a/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/jobs/IngestRemappedVariantsFromVcfJobConfigurationTest.java b/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/jobs/IngestRemappedVariantsFromVcfJobConfigurationTest.java index c585da7bf..232ec7218 100644 --- a/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/jobs/IngestRemappedVariantsFromVcfJobConfigurationTest.java +++ b/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/jobs/IngestRemappedVariantsFromVcfJobConfigurationTest.java @@ -15,65 +15,66 @@ */ package uk.ac.ebi.eva.remapping.ingest.configuration.batch.jobs; -import com.lordofthejars.nosqlunit.annotation.UsingDataSet; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; -import org.junit.After; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.batch.core.BatchStatus; import org.springframework.batch.core.JobExecution; import org.springframework.batch.core.StepExecution; import org.springframework.batch.test.JobLauncherTestUtils; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.ApplicationContext; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.core.io.ResourceLoader; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.data.mongodb.core.query.Criteria; import org.springframework.data.mongodb.core.query.Query; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; - +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; +import uk.ac.ebi.eva.accession.core.utils.MongoTestDataLoader; import uk.ac.ebi.eva.remapping.ingest.batch.tasklets.RemappingMetadata; +import uk.ac.ebi.eva.remapping.ingest.test.configuration.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.remapping.ingest.test.configuration.BatchTestConfiguration; -import uk.ac.ebi.eva.remapping.ingest.test.rule.FixSpringMongoDbRule; +import uk.ac.ebi.eva.remapping.ingest.test.configuration.MongoTestConfiguration; import java.util.Arrays; import java.util.Collection; import java.util.List; import java.util.stream.Collectors; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static uk.ac.ebi.eva.remapping.ingest.configuration.BeanNames.INGEST_REMAPPED_VARIANTS_FROM_VCF_STEP; import static uk.ac.ebi.eva.remapping.ingest.configuration.BeanNames.STORE_REMAPPING_METADATA_STEP; +import static uk.ac.ebi.eva.remapping.ingest.test.configuration.BatchTestConfiguration.JOB_INGEST_REMAPPED_VARIANTS_FROM_VCF_JOB; -@RunWith(SpringRunner.class) -@ContextConfiguration(classes = {BatchTestConfiguration.class}) +@ExtendWith(SpringExtension.class) +@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class, + BatchJobRepositoryTestConfiguration.class}) @TestPropertySource("classpath:ingest-remapped-variants.properties") -@UsingDataSet(locations = {"/test-data/submittedVariantEntity.json"}) -public class IngestRemappedVariantsFromVcfJobConfigurationTest { - - private static final String TEST_DB = "test-ingest-remapping"; +public class IngestRemappedVariantsFromVcfJobConfigurationTest extends MongoTestContainerHelper { @Autowired + @Qualifier(JOB_INGEST_REMAPPED_VARIANTS_FROM_VCF_JOB) private JobLauncherTestUtils jobLauncherTestUtils; @Autowired private MongoTemplate mongoTemplate; - //Required by nosql-unit @Autowired - private ApplicationContext applicationContext; + private ResourceLoader resourceLoader; - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName(TEST_DB).build()); + @BeforeEach + public void setUp() { + mongoTemplate.getDb().drop(); + new MongoTestDataLoader(mongoTemplate, resourceLoader).load("/test-data/submittedVariantEntity.json"); + } - @After + @AfterEach public void tearDown() { mongoTemplate.getDb().drop(); } @@ -83,7 +84,7 @@ public void tearDown() { public void jobFromVcf() throws Exception { JobExecution jobExecution = jobLauncherTestUtils.launchJob(); List expectedSteps = Arrays.asList(STORE_REMAPPING_METADATA_STEP, - INGEST_REMAPPED_VARIANTS_FROM_VCF_STEP); + INGEST_REMAPPED_VARIANTS_FROM_VCF_STEP); assertStepsExecuted(expectedSteps, jobExecution); assertEquals(BatchStatus.COMPLETED, jobExecution.getStatus()); assertMetadataAssociatedToSubmittedVariants(); @@ -104,9 +105,9 @@ private void assertMetadataAssociatedToSubmittedVariants() { List variantsRemapped = mongoTemplate.find(remappedQuery, SubmittedVariantEntity.class); long variantsWithMetatada = variantsRemapped.stream() - .filter(x -> x.getRemappingId() != null && - x.getRemappingId().equals(remappingId)) - .count(); + .filter(x -> x.getRemappingId() != null && + x.getRemappingId().equals(remappingId)) + .count(); assertEquals(4, variantsWithMetatada); } } \ No newline at end of file diff --git a/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/steps/IngestRemappedFromVcfStepConfigurationTest.java b/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/steps/IngestRemappedFromVcfStepConfigurationTest.java index f74407a71..931ba1175 100644 --- a/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/steps/IngestRemappedFromVcfStepConfigurationTest.java +++ b/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/steps/IngestRemappedFromVcfStepConfigurationTest.java @@ -16,75 +16,60 @@ */ package uk.ac.ebi.eva.remapping.ingest.configuration.batch.steps; -import com.lordofthejars.nosqlunit.annotation.UsingDataSet; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.batch.core.BatchStatus; import org.springframework.batch.core.JobExecution; import org.springframework.batch.test.JobLauncherTestUtils; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.ApplicationContext; +import org.springframework.core.io.ResourceLoader; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.data.mongodb.core.query.Criteria; import org.springframework.data.mongodb.core.query.Query; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; -import uk.ac.ebi.ampt2d.commons.accession.hashing.SHA1HashingFunction; - -import uk.ac.ebi.eva.accession.core.model.ISubmittedVariant; +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; -import uk.ac.ebi.eva.accession.core.summary.SubmittedVariantSummaryFunction; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; +import uk.ac.ebi.eva.accession.core.utils.MongoTestDataLoader; import uk.ac.ebi.eva.remapping.ingest.configuration.BeanNames; +import uk.ac.ebi.eva.remapping.ingest.test.configuration.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.remapping.ingest.test.configuration.BatchTestConfiguration; -import uk.ac.ebi.eva.remapping.ingest.test.rule.FixSpringMongoDbRule; +import uk.ac.ebi.eva.remapping.ingest.test.configuration.MongoTestConfiguration; import java.util.List; -import java.util.function.Function; -import static junit.framework.TestCase.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; -@RunWith(SpringRunner.class) -@ContextConfiguration(classes = {BatchTestConfiguration.class}) +@ExtendWith(SpringExtension.class) +@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class, + BatchJobRepositoryTestConfiguration.class}) @TestPropertySource("classpath:ingest-remapped-variants.properties") -@UsingDataSet(locations = {"/test-data/submittedVariantEntity.json"}) -public class IngestRemappedFromVcfStepConfigurationTest { - - private static final String TEST_DB = "test-ingest-remapping"; +public class IngestRemappedFromVcfStepConfigurationTest extends MongoTestContainerHelper { private static final String SUBMITTED_VARIANT_COLLECTION = "submittedVariantEntity"; public static final String REMAPPED_FROM = "GCA_000000001.1"; - private Function hashingFunction; - @Autowired private JobLauncherTestUtils jobLauncherTestUtils; @Autowired private MongoTemplate mongoTemplate; - //Required by nosql-unit @Autowired - private ApplicationContext applicationContext; + private ResourceLoader resourceLoader; - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName(TEST_DB).build()); - - @Before - public void setUp() throws Exception { - hashingFunction = new SubmittedVariantSummaryFunction().andThen(new SHA1HashingFunction()); + @BeforeEach + public void setUp() { + mongoTemplate.getDb().drop(); + new MongoTestDataLoader(mongoTemplate, resourceLoader).load("/test-data/submittedVariantEntity.json"); } - @After + @AfterEach public void tearDown() { mongoTemplate.getDb().drop(); } @@ -96,14 +81,14 @@ public void runStep() { assertEquals(7, mongoTemplate.getCollection(SUBMITTED_VARIANT_COLLECTION).countDocuments()); JobExecution jobExecution = jobLauncherTestUtils.launchStep(BeanNames.INGEST_REMAPPED_VARIANTS_FROM_VCF_STEP); - Assert.assertEquals(BatchStatus.COMPLETED, jobExecution.getStatus()); + assertEquals(BatchStatus.COMPLETED, jobExecution.getStatus()); //Documents in the database after the ingestion assertEquals(11, mongoTemplate.getCollection(SUBMITTED_VARIANT_COLLECTION).countDocuments()); Query remappedVariantsQuery = new Query(Criteria.where("remappedFrom").is(REMAPPED_FROM)); List remappedVariants = mongoTemplate.find(remappedVariantsQuery, - SubmittedVariantEntity.class); + SubmittedVariantEntity.class); assertEquals(6, remappedVariants.size()); diff --git a/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/steps/IngestRemappedFromVcfStepDbsnpConfigurationTest.java b/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/steps/IngestRemappedFromVcfStepDbsnpConfigurationTest.java index d53ce1a8f..377c3ee6d 100644 --- a/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/steps/IngestRemappedFromVcfStepDbsnpConfigurationTest.java +++ b/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/steps/IngestRemappedFromVcfStepDbsnpConfigurationTest.java @@ -16,87 +16,79 @@ */ package uk.ac.ebi.eva.remapping.ingest.configuration.batch.steps; -import com.lordofthejars.nosqlunit.annotation.UsingDataSet; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.batch.core.BatchStatus; import org.springframework.batch.core.JobExecution; import org.springframework.batch.test.JobLauncherTestUtils; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.ApplicationContext; +import org.springframework.core.io.ResourceLoader; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.data.mongodb.core.query.Criteria; import org.springframework.data.mongodb.core.query.Query; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; -import uk.ac.ebi.ampt2d.commons.accession.hashing.SHA1HashingFunction; - -import uk.ac.ebi.eva.accession.core.model.ISubmittedVariant; +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpSubmittedVariantEntity; -import uk.ac.ebi.eva.accession.core.summary.SubmittedVariantSummaryFunction; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; +import uk.ac.ebi.eva.accession.core.utils.MongoTestDataLoader; import uk.ac.ebi.eva.remapping.ingest.configuration.BeanNames; +import uk.ac.ebi.eva.remapping.ingest.test.configuration.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.remapping.ingest.test.configuration.BatchTestConfiguration; -import uk.ac.ebi.eva.remapping.ingest.test.rule.FixSpringMongoDbRule; +import uk.ac.ebi.eva.remapping.ingest.test.configuration.MongoTestConfiguration; import java.util.List; -import java.util.function.Function; -import static junit.framework.TestCase.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; -@RunWith(SpringRunner.class) -@ContextConfiguration(classes = {BatchTestConfiguration.class}) +@ExtendWith(SpringExtension.class) +@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class, + BatchJobRepositoryTestConfiguration.class}) @TestPropertySource("classpath:ingest-remapped-variants-dbsnp.properties") -@UsingDataSet(locations = {"/test-data/dbsnpSubmittedVariantEntity.json"}) -public class IngestRemappedFromVcfStepDbsnpConfigurationTest { - - private static final String TEST_DB = "test-ingest-remapping"; +public class IngestRemappedFromVcfStepDbsnpConfigurationTest extends MongoTestContainerHelper { private static final String DBSNP_SUBMITTED_VARIANT_COLLECTION = "dbsnpSubmittedVariantEntity"; public static final String REMAPPED_FROM = "GCA_000000001.1"; - private Function hashingFunction; - @Autowired private JobLauncherTestUtils jobLauncherTestUtils; @Autowired private MongoTemplate mongoTemplate; - //Required by nosql-unit @Autowired - private ApplicationContext applicationContext; + private ResourceLoader resourceLoader; - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName(TEST_DB).build()); + @BeforeEach + public void setUp() { + mongoTemplate.getDb().drop(); + new MongoTestDataLoader(mongoTemplate, resourceLoader).load("/test-data/dbsnpSubmittedVariantEntity.json"); + } - @Before - public void setUp() throws Exception { - hashingFunction = new SubmittedVariantSummaryFunction().andThen(new SHA1HashingFunction()); + @AfterEach + public void tearDown() { + mongoTemplate.getDb().drop(); } + @Test @DirtiesContext public void runStepDbsnp() { assertEquals(1, mongoTemplate.getCollection(DBSNP_SUBMITTED_VARIANT_COLLECTION).countDocuments()); JobExecution jobExecution = jobLauncherTestUtils.launchStep(BeanNames.INGEST_REMAPPED_VARIANTS_FROM_VCF_STEP); - Assert.assertEquals(BatchStatus.COMPLETED, jobExecution.getStatus()); + assertEquals(BatchStatus.COMPLETED, jobExecution.getStatus()); //Documents in the database after the ingestion assertEquals(2, mongoTemplate.getCollection(DBSNP_SUBMITTED_VARIANT_COLLECTION).countDocuments()); Query remappedVariantsQuery = new Query(Criteria.where("remappedFrom").is(REMAPPED_FROM)); List remappedVariants = mongoTemplate.find(remappedVariantsQuery, - DbsnpSubmittedVariantEntity.class); + DbsnpSubmittedVariantEntity.class); assertEquals(1, remappedVariants.size()); diff --git a/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/test/configuration/BatchJobRepositoryTestConfiguration.java b/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/test/configuration/BatchJobRepositoryTestConfiguration.java new file mode 100644 index 000000000..cfa3352cb --- /dev/null +++ b/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/test/configuration/BatchJobRepositoryTestConfiguration.java @@ -0,0 +1,22 @@ +package uk.ac.ebi.eva.remapping.ingest.test.configuration; + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Primary; +import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseBuilder; +import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseType; + +import javax.sql.DataSource; + +@Configuration +public class BatchJobRepositoryTestConfiguration { + + @Bean + @Primary + public DataSource dataSource() { + return new EmbeddedDatabaseBuilder() + .setType(EmbeddedDatabaseType.HSQL) + .addScript("org/springframework/batch/core/schema-hsqldb.sql") + .build(); + } +} \ No newline at end of file diff --git a/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/test/configuration/BatchTestConfiguration.java b/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/test/configuration/BatchTestConfiguration.java index 9471b8897..a7065192a 100644 --- a/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/test/configuration/BatchTestConfiguration.java +++ b/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/test/configuration/BatchTestConfiguration.java @@ -16,15 +16,16 @@ */ package uk.ac.ebi.eva.remapping.ingest.test.configuration; +import org.springframework.batch.core.Job; +import org.springframework.batch.core.launch.JobLauncher; +import org.springframework.batch.core.repository.JobRepository; import org.springframework.batch.test.JobLauncherTestUtils; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; -import org.springframework.boot.autoconfigure.batch.BatchProperties; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Import; import org.springframework.core.io.ResourceLoader; -import org.springframework.transaction.PlatformTransactionManager; - import uk.ac.ebi.eva.commons.batch.job.JobExecutionApplicationListener; import uk.ac.ebi.eva.remapping.ingest.configuration.RemappingMetadataConfiguration; import uk.ac.ebi.eva.remapping.ingest.configuration.batch.io.IngestRemappedSubmittedVariantsWriterConfiguration; @@ -36,7 +37,7 @@ import uk.ac.ebi.eva.remapping.ingest.configuration.batch.steps.StoreRemappingMetadataStepConfiguration; import uk.ac.ebi.eva.remapping.ingest.configuration.policies.ChunkSizeCompletionPolicyConfiguration; -import javax.sql.DataSource; +import static uk.ac.ebi.eva.remapping.ingest.configuration.BeanNames.INGEST_REMAPPED_VARIANTS_FROM_VCF_JOB; @EnableAutoConfiguration @Import({IngestRemappedFromVcfStepConfiguration.class, @@ -49,22 +50,19 @@ ChunkSizeCompletionPolicyConfiguration.class, RemappingMetadataConfiguration.class}) public class BatchTestConfiguration { - - @Autowired - private BatchProperties properties; + public static final String JOB_INGEST_REMAPPED_VARIANTS_FROM_VCF_JOB = "JOB_INGEST_REMAPPED_VARIANTS_FROM_VCF_JOB"; @Autowired private ResourceLoader resourceLoader; - @Autowired - private DataSource dataSource; - - @Autowired - private PlatformTransactionManager platformTransactionManager; - - @Bean - public JobLauncherTestUtils jobLauncherTestUtils() { - return new JobLauncherTestUtils(); + @Bean(JOB_INGEST_REMAPPED_VARIANTS_FROM_VCF_JOB) + public JobLauncherTestUtils jobLauncherTestUtilsFromMongo(JobLauncher jobLauncher, JobRepository jobRepository, + @Qualifier(INGEST_REMAPPED_VARIANTS_FROM_VCF_JOB) Job job) { + JobLauncherTestUtils utils = new JobLauncherTestUtils(); + utils.setJobLauncher(jobLauncher); + utils.setJobRepository(jobRepository); + utils.setJob(job); + return utils; } @Bean diff --git a/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/test/configuration/MongoTestConfiguration.java b/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/test/configuration/MongoTestConfiguration.java index 95135e52c..2cab74199 100644 --- a/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/test/configuration/MongoTestConfiguration.java +++ b/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/test/configuration/MongoTestConfiguration.java @@ -15,12 +15,30 @@ */ package uk.ac.ebi.eva.remapping.ingest.test.configuration; +import com.mongodb.ReadPreference; +import com.mongodb.WriteConcern; +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoClients; +import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.domain.EntityScan; import org.springframework.boot.test.autoconfigure.data.mongo.AutoConfigureDataMongo; +import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Primary; +import org.springframework.data.mongodb.MongoDatabaseFactory; import org.springframework.data.mongodb.config.EnableMongoAuditing; +import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.data.mongodb.core.SimpleMongoClientDatabaseFactory; +import org.springframework.data.mongodb.core.WriteResultChecking; +import org.springframework.data.mongodb.core.convert.DefaultDbRefResolver; +import org.springframework.data.mongodb.core.convert.DefaultMongoTypeMapper; +import org.springframework.data.mongodb.core.convert.MappingMongoConverter; +import org.springframework.data.mongodb.core.convert.MongoCustomConversions; +import org.springframework.data.mongodb.core.mapping.MongoMappingContext; import org.springframework.data.mongodb.repository.config.EnableMongoRepositories; +import java.util.Collections; + @Configuration @EntityScan(basePackages = {"uk.ac.ebi.eva.accession.core.persistence"}) @EnableMongoRepositories(basePackages = { @@ -29,4 +47,56 @@ @EnableMongoAuditing @AutoConfigureDataMongo public class MongoTestConfiguration { + + @Value("${spring.data.mongodb.uri}") + private String mongoUri; + + @Value("${spring.data.mongodb.database}") + private String database; + + @Value("${mongodb.read-preference:primary}") + private String readPreference; + + @Bean + @Primary + public MongoClient mongoClient() { + return MongoClients.create(mongoUri); + } + + @Bean("primaryFactory") + @Primary + public MongoDatabaseFactory mongoDbFactory() { + return new SimpleMongoClientDatabaseFactory(mongoClient(), database); + } + + @Bean + @Primary + public MongoCustomConversions mongoCustomConversions() { + return new MongoCustomConversions(Collections.emptyList()); + } + + @Bean + @Primary + public MongoMappingContext mongoMappingContext(MongoCustomConversions conversions) { + MongoMappingContext context = new MongoMappingContext(); + context.setSimpleTypeHolder(conversions.getSimpleTypeHolder()); + context.setAutoIndexCreation(false); + return context; + } + + @Bean + @Primary + public MongoTemplate mongoTemplate(MongoMappingContext mongoMappingContext, + MongoCustomConversions conversions) { + MappingMongoConverter converter = new MappingMongoConverter( + new DefaultDbRefResolver(mongoDbFactory()), mongoMappingContext); + converter.setCustomConversions(conversions); + converter.setTypeMapper(new DefaultMongoTypeMapper(null)); + converter.afterPropertiesSet(); + MongoTemplate mongoTemplate = new MongoTemplate(mongoDbFactory(), converter); + mongoTemplate.setReadPreference(ReadPreference.valueOf(readPreference)); + mongoTemplate.setWriteConcern(WriteConcern.MAJORITY); + mongoTemplate.setWriteResultChecking(WriteResultChecking.EXCEPTION); + return mongoTemplate; + } } diff --git a/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/test/rule/FixSpringMongoDbRule.java b/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/test/rule/FixSpringMongoDbRule.java deleted file mode 100644 index 9386f9b1d..000000000 --- a/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/test/rule/FixSpringMongoDbRule.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright 2021 EMBL - European Bioinformatics Institute - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package uk.ac.ebi.eva.remapping.ingest.test.rule; - -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfiguration; -import com.lordofthejars.nosqlunit.mongodb.SpringMongoDbRule; - -/** - * Temporary fix until nosql unit rc-6 or final is released - */ -public class FixSpringMongoDbRule extends SpringMongoDbRule { - - public FixSpringMongoDbRule(MongoDbConfiguration mongoDbConfiguration) { - super(mongoDbConfiguration); - } - - public FixSpringMongoDbRule(MongoDbConfiguration mongoDbConfiguration, Object object) { - super(mongoDbConfiguration, object); - } - - @Override - public void close() { - // DO NOT CLOSE the connection (Spring will do it when destroying the context) - } -} diff --git a/eva-remapping-ingest/src/test/resources/ingest-remapped-variants-dbsnp.properties b/eva-remapping-ingest/src/test/resources/ingest-remapped-variants-dbsnp.properties index a4baca6b4..07145cff2 100644 --- a/eva-remapping-ingest/src/test/resources/ingest-remapped-variants-dbsnp.properties +++ b/eva-remapping-ingest/src/test/resources/ingest-remapped-variants-dbsnp.properties @@ -10,9 +10,6 @@ parameters.chunkSize=2 build.version=|project.version| spring.data.mongodb.database=test-ingest-remapping -spring.data.mongodb.host=|eva.mongo.host.test| -spring.data.mongodb.password= -spring.data.mongodb.port=27017 mongodb.read-preference=primary # See https://github.com/spring-projects/spring-boot/wiki/Spring-Boot-2.1-Release-Notes#bean-overriding diff --git a/eva-remapping-ingest/src/test/resources/ingest-remapped-variants.properties b/eva-remapping-ingest/src/test/resources/ingest-remapped-variants.properties index c2ee2a787..864bdf764 100644 --- a/eva-remapping-ingest/src/test/resources/ingest-remapped-variants.properties +++ b/eva-remapping-ingest/src/test/resources/ingest-remapped-variants.properties @@ -10,9 +10,6 @@ parameters.chunkSize=2 build.version=|project.version| spring.data.mongodb.database=test-ingest-remapping -spring.data.mongodb.host=|eva.mongo.host.test| -spring.data.mongodb.password= -spring.data.mongodb.port=27017 mongodb.read-preference=primary # See https://github.com/spring-projects/spring-boot/wiki/Spring-Boot-2.1-Release-Notes#bean-overriding diff --git a/pom.xml b/pom.xml index 96135f75c..c0f2ee9ea 100644 --- a/pom.xml +++ b/pom.xml @@ -1,10 +1,9 @@ 4.0.0 - uk.ac.ebi.eva eva-accession - 0.6.57-SNAPSHOT + ${revision} pom eva-accession @@ -23,16 +22,20 @@ + 0.6.57-SNAPSHOT + 21 UTF-8 - 4.13 - 0.8.10 - 0.7.21 + 1.0.0-SNAPSHOT + 1.0.0-SNAPSHOT + 4.0.0 + 1.0.0-SNAPSHOT + 2.8.17 org.springframework.boot spring-boot-starter-parent - 2.1.0.RELEASE + 3.4.13 @@ -40,18 +43,49 @@ uk.ac.ebi.eva eva-accession-core - 0.6.57-SNAPSHOT + ${revision} uk.ac.ebi.eva - eva-accession-clustering - 0.6.57-SNAPSHOT + eva-accession-ws + ${revision} + + + uk.ac.ebi.eva + eva-accession-pipeline + ${revision} + + + uk.ac.ebi.eva + eva-accession-release + ${revision} uk.ac.ebi.eva eva-remapping-get-source - 0.6.57-SNAPSHOT + ${revision} + + + uk.ac.ebi.eva + eva-accession-deprecate + ${revision} + + + uk.ac.ebi.eva + eva-accession-import-dbsnp2 + ${revision} + + uk.ac.ebi.eva + eva-accession-clustering + ${revision} + + + uk.ac.ebi.eva + eva-remapping-ingest + ${revision} + + uk.ac.ebi.ampt2d accession-commons-mongodb @@ -85,38 +119,22 @@ uk.ac.ebi.eva eva-metrics - 0.0.4 - - - org.postgresql - postgresql - 42.7.5 - - - com.lordofthejars - nosqlunit-mongodb - 1.0.0-rc.5 - test + ${eva-metrics-version} com.github.samtools htsjdk - 2.14.3 + ${samtools-htsjdk-version} com.thoughtworks.xstream xstream - 1.4.9 + 1.4.21 - io.springfox - springfox-swagger2 - 2.8.0 - - - io.springfox - springfox-swagger-ui - 2.8.0 + org.springdoc + springdoc-openapi-starter-webmvc-ui + ${springdoc.version} @@ -135,10 +153,30 @@ test + + org.postgresql + postgresql + + org.hsqldb hsqldb - 2.3.4 + test + + + + org.springframework.boot + spring-boot-testcontainers + test + + + org.testcontainers + mongodb + test + + + org.testcontainers + junit-jupiter test @@ -170,11 +208,27 @@ + + + + org.apache.maven.plugins + maven-surefire-plugin + + false + + --add-opens java.base/java.time=ALL-UNNAMED + --add-opens java.base/java.lang=ALL-UNNAMED + --add-opens java.base/java.util=ALL-UNNAMED + + + + + org.apache.maven.plugins maven-resources-plugin - 2.6 + 3.3.0 @@ -186,21 +240,16 @@ org.apache.maven.plugins maven-compiler-plugin - 1.8 - 1.8 - - - - org.apache.maven.plugins - maven-surefire-plugin - - false + ${java.version} + true + true + UTF-8 - pl.project13.maven - git-commit-id-plugin - 2.2.5 + io.github.git-commit-id + git-commit-id-maven-plugin + 9.0.1 ^git.tags$ @@ -212,7 +261,32 @@ + + org.codehaus.mojo + flatten-maven-plugin + 1.6.0 + + true + resolveCiFriendliesOnly + + + + flatten + process-resources + + flatten + + + + flatten.clean + clean + + clean + + + + - + \ No newline at end of file From 8aee4872fbc733e2901b9c07bd96d3204e9d9485 Mon Sep 17 00:00:00 2001 From: nitin-ebi <79518737+nitin-ebi@users.noreply.github.com> Date: Thu, 4 Jun 2026 15:08:34 +0100 Subject: [PATCH 04/10] EVA-4177 Remove external job tracker - use in memory one (#502) --- eva-accession-clustering/pom.xml | 11 + .../eva/accession/clustering/Application.java | 19 +- .../jobs/BackPropagateRSJobConfiguration.java | 5 - ...erUnclusteredVariantsJobConfiguration.java | 4 - .../ClusteringFromMongoJobConfiguration.java | 11 +- ...emappedVariantsWithRSJobConfiguration.java | 4 - .../RSAccessionRecoveryJobConfiguration.java | 4 - .../jobs/StudyClusteringJobConfiguration.java | 4 - .../jobs/qc/ClusteringQCJobConfiguration.java | 11 +- .../qc/DuplicateRSAccQCJobConfiguration.java | 4 - ...ewClusteredVariantsQCJobConfiguration.java | 9 +- ...AccessioningShutdownStepConfiguration.java | 7 +- .../ClusteringFromMongoStepConfiguration.java | 27 ++- .../RSAccessionRecoveryStepConfiguration.java | 4 +- .../qc/DuplicateRSAccQCStepConfiguration.java | 6 +- .../parameters/InputParameters.java | 10 - .../runner/ClusteringCommandLineRunner.java | 15 +- .../src/main/resources/application.properties | 6 - .../batch/io/BackPropagatedRSWriterTest.java | 6 +- .../batch/io/ClusteringMongoReaderTest.java | 2 +- .../RSMergeAndSplitCandidatesReaderTest.java | 7 +- .../batch/io/RSMergeWriterTest.java | 5 +- .../batch/io/RSSplitWriterTest.java | 26 ++- .../batch/io/SSSplitWriterTest.java | 24 +- .../io/StudyClusteringMongoReaderTest.java | 2 +- .../IssueAccessionClusteringWriterTest.java | 27 ++- .../MergeAccessionClusteringWriterTest.java | 8 +- .../RemappedVariantsClusteringWriterTest.java | 27 ++- .../ReuseAccessionClusteringWriterTest.java | 27 ++- ...ClusteringVariantJobConfigurationTest.java | 22 +- .../batch/jobs/RSAccessionRecoveryTest.java | 35 ++- ...DuplicateRSAccQCStepConfigurationTest.java | 26 ++- ...usteredVariantsQCJobConfigurationTest.java | 7 +- ...lusteringVariantStepConfigurationTest.java | 22 +- .../ClusteringCommandLineRunnerTest.java | 94 ++------ .../BatchJobRepositoryTestConfiguration.java | 22 -- .../configuration/BatchTestConfiguration.java | 13 +- .../configuration/MongoTestConfiguration.java | 102 --------- .../RSAccessionRecoveryTestConfiguration.java | 2 - .../resources/backpropagation-test.properties | 4 + .../clustering-issuance-test.properties | 5 + .../clustering-pipeline-test.properties | 5 + .../resources/clustering-qc-test.properties | 5 + .../clustering-writer-test.properties | 5 + .../duplicate-rs-acc-qc-test.properties | 5 + .../resources/merge-split-test.properties | 5 + .../rs-accession-recovery.properties | 12 +- .../test-data/contiguous_id_blocks_schema.sql | 4 + eva-accession-core/pom.xml | 9 +- .../accession/core/GenericApplication.java | 15 +- ...iguousIdBlocksDataSourceConfiguration.java | 50 ++++ .../InMemoryBatchConfiguration.java | 50 ++++ .../core/configuration/NoOpJobExplorer.java | 65 ++++++ .../core/runner/CommandLineRunnerUtils.java | 96 -------- ...ClusteredVariantDeprecationWriterTest.java | 6 +- ...SubmittedVariantDeprecationWriterTest.java | 8 +- ...ClusteredVariantEntityIntegrationTest.java | 2 - ...SubmittedVariantEntityIntegrationTest.java | 2 - ...ttedVariantAccessioningRepositoryTest.java | 4 +- ...usteredVariantAccessioningServiceTest.java | 4 +- ...bmittedVariantAccessioningServiceTest.java | 26 ++- .../configuration/JPATestConfiguration.java | 41 ---- .../properties/ss-accession-test.properties | 5 + .../properties/ss-deprecation-test.properties | 5 + .../test-data/contiguous_id_blocks_schema.sql | 14 ++ eva-accession-deprecate/pom.xml | 11 + .../eva/accession/deprecate/Application.java | 19 +- ...tudySubmittedVariantsJobConfiguration.java | 4 - ...ittedVariantsFromFileJobConfiguration.java | 4 - ...udySubmittedVariantsStepConfiguration.java | 5 +- ...ttedVariantsFromFileStepConfiguration.java | 5 +- .../src/main/resources/application.properties | 6 - .../io/StudySubmittedVariantsReaderTest.java | 2 +- ...SubmittedVariantsJobConfigurationTest.java | 37 +-- ...dVariantsFromFileJobConfigurationTest.java | 37 +-- ...ubmittedVariantsStepConfigurationTest.java | 6 +- ...VariantsFromFileStepConfigurationTest.java | 6 +- .../BatchJobRepositoryTestConfiguration.java | 22 -- .../configuration/BatchTestConfiguration.java | 14 +- .../configuration/MongoTestConfiguration.java | 105 --------- ...bmitted-variants-from-file-test.properties | 5 + .../study-submitted-variants-test.properties | 5 + eva-accession-import-dbsnp2/pom.xml | 10 +- .../ebi/eva/accession/dbsnp2/Application.java | 16 +- ...portDbsnpJsonVariantsJobConfiguration.java | 4 - ...ortDbsnpJsonVariantsStepConfiguration.java | 6 +- .../dbsnp2/parameters/InputParameters.java | 19 +- ...tVariantsJobLauncherCommandLineRunner.java | 16 +- .../src/main/resources/application.properties | 1 - .../DbsnpJsonClusteredVariantsWriterTest.java | 2 +- ...DbsnpJsonVariantsJobConfigurationTest.java | 6 +- ...bsnpJsonVariantsStepConfigurationTest.java | 6 +- ...iantsJobLauncherCommandLineRunnerTest.java | 139 +----------- .../BatchJobRepositoryTestConfiguration.java | 22 -- .../dbsnp2/test/BatchTestConfiguration.java | 8 +- .../dbsnp2/test/MongoTestConfiguration.java | 102 --------- .../src/test/resources/application.properties | 1 - eva-accession-pipeline/pom.xml | 5 + .../eva/accession/pipeline/Application.java | 20 +- .../DuplicateSSAccQCJobConfiguration.java | 4 - .../QCSubsnpAccessionsJobConfiguration.java | 4 - .../SSAccessionRecoveryJobConfiguration.java | 4 - .../SubsnpAccessionsJobConfiguration.java | 4 - ...AccessioningShutdownStepConfiguration.java | 7 +- .../steps/BuildReportStepConfiguration.java | 7 +- .../DuplicateSSAccQCStepConfiguration.java | 6 +- .../QCSubsnpAccessionsStepConfiguration.java | 6 +- .../SSAccessionRecoveryStepConfiguration.java | 2 - .../SubsnpAccessionsStepConfiguration.java | 6 +- .../pipeline/parameters/InputParameters.java | 10 - ...AccessionJobLauncherCommandLineRunner.java | 16 +- .../src/main/resources/application.properties | 6 - .../batch/io/AccessionWriterTest.java | 25 +- ...fSubsnpAccessionsJobConfigurationTest.java | 22 +- ...eSubsnpAccessionsJobConfigurationTest.java | 22 +- ...reateSubsnpAccessionsRecoverStateTest.java | 31 ++- ...nsRecoveringStateJobConfigurationTest.java | 24 +- .../DuplicateSSAccQCJobConfigurationTest.java | 6 +- ...CSubsnpAccessionsJobConfigurationTest.java | 6 +- ...SubsnpAccessionsStepConfigurationTest.java | 22 +- ...SubsnpAccessionsStepConfigurationTest.java | 6 +- ...ssionJobLauncherCommandLineRunnerTest.java | 165 ++------------ .../runner/JobFailureBlocksReleasedTest.java | 23 +- .../pipeline/runner/RestartFailedJobTest.java | 214 ------------------ .../runner/SSAccessionRecoveryTest.java | 38 +++- .../BatchJobRepositoryTestConfiguration.java | 22 -- .../pipeline/test/BatchTestConfiguration.java | 11 +- .../pipeline/test/MongoTestConfiguration.java | 102 --------- .../SSAccessionRecoveryTestConfiguration.java | 2 - ...ccession-pipeline-interval-test.properties | 5 + ...ion-pipeline-recover-state-test.properties | 12 +- ...accession-pipeline-recover-test.properties | 12 +- .../accession-pipeline-test.properties | 5 + .../duplicate-ss-acc-qc-test.properties | 5 + .../qc-accession-pipeline-test.properties | 5 + .../ss-accession-recovery.properties | 12 +- .../test-data/contiguous_id_blocks_schema.sql | 6 +- .../eva/accession/release/Application.java | 11 +- .../configuration/RunnerConfiguration.java | 25 ++ ...ccessionReleaseFromDBJobConfiguration.java | 4 - .../DumpRSAccessionsJobConfiguration.java | 5 - ...ccessionReleaseFromDBJobConfiguration.java | 4 - ...cessionReleaseFromDBStepConfiguration.java | 7 +- .../DumpRSAccessionsStepConfiguration.java | 9 +- ...cessionReleaseFromDBStepConfiguration.java | 7 +- .../release/parameters/InputParameters.java | 10 - .../src/main/resources/application.properties | 1 - .../AccessionedVariantMongoReaderTest.java | 2 +- .../io/contig/ContigMongoReaderTest.java | 2 +- .../DeprecatedVariantMongoReaderTest.java | 2 +- .../merged/MergedVariantMongoReaderTest.java | 2 +- ...ergedDeprecatedVariantMongoReaderTest.java | 2 +- .../batch/steps/DumpRSAccessionsTest.java | 8 +- .../steps/RSAccessionsReleaseFromDBTest.java | 8 +- .../BatchJobRepositoryTestConfiguration.java | 22 -- .../configuration/BatchTestConfiguration.java | 2 - .../configuration/MongoTestConfiguration.java | 105 --------- .../ReleaseFromDBTestConfiguration.java | 3 - .../src/test/resources/application.properties | 1 - .../dump-rs-accession-test.properties | 1 - ...cremental-release-pipeline-test.properties | 27 --- .../release-pipeline-test.properties | 28 --- eva-accession-ws/pom.xml | 6 + .../accession/ws/EvaAccessionApplication.java | 16 +- .../ws/rest/EvaControllerAdvice.java | 2 +- .../src/main/resources/application.properties | 5 + .../ClusteredVariantHistoryEndPointTest.java | 5 +- .../ClusteredVariantsRestControllerTest.java | 5 +- .../SubmittedVariantsRestControllerTest.java | 28 ++- .../ws/test/MongoTestConfiguration.java | 105 --------- .../resources/accession-ws-test.properties | 5 + .../test-data/contiguous_id_blocks_schema.sql | 14 ++ .../ebi/eva/remapping/source/Application.java | 18 +- ...portSubmittedVariantsJobConfiguration.java | 17 +- ...ortSubmittedVariantsStepConfiguration.java | 8 +- .../source/parameters/InputParameters.java | 10 - ...RemappingJobLauncherCommandLineRunner.java | 16 +- .../src/main/resources/application.properties | 9 - .../DbsnpSubmittedVariantMongoReaderTest.java | 2 +- .../EvaSubmittedVariantMongoReaderTest.java | 2 +- ...SubmittedVariantsJobConfigurationTest.java | 6 +- ...ppingJobLauncherCommandLineRunnerTest.java | 6 +- ...cherCommandLineRunnerWithProjectsTest.java | 6 +- .../BatchJobRepositoryTestConfiguration.java | 22 -- .../configuration/BatchTestConfiguration.java | 6 +- .../configuration/MongoTestConfiguration.java | 105 --------- .../src/test/resources/application.properties | 1 - .../test/resources/with-projects.properties | 1 - .../ebi/eva/remapping/ingest/Application.java | 18 +- ...mappedVariantsFromVcfJobConfiguration.java | 4 - ...ngestRemappedFromVcfStepConfiguration.java | 6 +- ...oreRemappingMetadataStepConfiguration.java | 9 +- .../ingest/parameters/InputParameters.java | 10 - .../IngestRemappedCommandLineRunner.java | 16 +- .../RemappedSubmittedVariantsWriterTest.java | 2 +- ...edVariantsFromVcfJobConfigurationTest.java | 6 +- ...tRemappedFromVcfStepConfigurationTest.java | 6 +- ...ppedFromVcfStepDbsnpConfigurationTest.java | 6 +- .../BatchJobRepositoryTestConfiguration.java | 22 -- .../configuration/BatchTestConfiguration.java | 6 +- .../configuration/MongoTestConfiguration.java | 102 --------- pom.xml | 11 - 202 files changed, 1207 insertions(+), 2421 deletions(-) delete mode 100644 eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/test/configuration/BatchJobRepositoryTestConfiguration.java delete mode 100644 eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/test/configuration/MongoTestConfiguration.java create mode 100644 eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/configuration/ContiguousIdBlocksDataSourceConfiguration.java create mode 100644 eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/configuration/InMemoryBatchConfiguration.java create mode 100644 eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/configuration/NoOpJobExplorer.java delete mode 100644 eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/runner/CommandLineRunnerUtils.java delete mode 100644 eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/test/configuration/JPATestConfiguration.java create mode 100644 eva-accession-core/src/test/resources/test-data/contiguous_id_blocks_schema.sql delete mode 100644 eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/test/configuration/BatchJobRepositoryTestConfiguration.java delete mode 100644 eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/test/configuration/MongoTestConfiguration.java delete mode 100644 eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/test/BatchJobRepositoryTestConfiguration.java delete mode 100644 eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/test/MongoTestConfiguration.java delete mode 100644 eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/runner/RestartFailedJobTest.java delete mode 100644 eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/test/BatchJobRepositoryTestConfiguration.java delete mode 100644 eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/test/MongoTestConfiguration.java delete mode 100644 eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/test/configuration/BatchJobRepositoryTestConfiguration.java delete mode 100644 eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/test/configuration/MongoTestConfiguration.java delete mode 100644 eva-accession-release/src/test/resources/incremental-release-pipeline-test.properties delete mode 100644 eva-accession-release/src/test/resources/release-pipeline-test.properties delete mode 100644 eva-accession-ws/src/test/java/uk/ac/ebi/eva/accession/ws/test/MongoTestConfiguration.java create mode 100644 eva-accession-ws/src/test/resources/test-data/contiguous_id_blocks_schema.sql delete mode 100644 eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/test/configuration/BatchJobRepositoryTestConfiguration.java delete mode 100644 eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/test/configuration/MongoTestConfiguration.java delete mode 100644 eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/test/configuration/BatchJobRepositoryTestConfiguration.java delete mode 100644 eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/test/configuration/MongoTestConfiguration.java diff --git a/eva-accession-clustering/pom.xml b/eva-accession-clustering/pom.xml index 53ac31e80..d699bec1f 100644 --- a/eva-accession-clustering/pom.xml +++ b/eva-accession-clustering/pom.xml @@ -36,6 +36,11 @@ gson + + org.postgresql + postgresql + + com.thoughtworks.xstream xstream @@ -61,6 +66,12 @@ test + + org.hsqldb + hsqldb + test + + org.springframework.boot spring-boot-testcontainers diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/Application.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/Application.java index 0f009be40..9e135c0f8 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/Application.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/Application.java @@ -15,13 +15,26 @@ */ package uk.ac.ebi.eva.accession.clustering; -import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.autoconfigure.batch.BatchAutoConfiguration; +import org.springframework.boot.autoconfigure.data.mongo.MongoDataAutoConfiguration; +import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration; +import org.springframework.boot.autoconfigure.jdbc.DataSourceTransactionManagerAutoConfiguration; +import org.springframework.boot.autoconfigure.orm.jpa.HibernateJpaAutoConfiguration; import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.context.annotation.Import; +import uk.ac.ebi.eva.accession.core.configuration.ContiguousIdBlocksDataSourceConfiguration; +import uk.ac.ebi.eva.accession.core.configuration.InMemoryBatchConfiguration; -@EnableBatchProcessing -@SpringBootApplication +@SpringBootApplication(exclude = { + MongoDataAutoConfiguration.class, + DataSourceAutoConfiguration.class, + DataSourceTransactionManagerAutoConfiguration.class, + HibernateJpaAutoConfiguration.class, + BatchAutoConfiguration.class +}) +@Import({InMemoryBatchConfiguration.class, ContiguousIdBlocksDataSourceConfiguration.class}) public class Application { public static void main(String[] args) { diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/BackPropagateRSJobConfiguration.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/BackPropagateRSJobConfiguration.java index ec9f999b8..620d4c892 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/BackPropagateRSJobConfiguration.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/BackPropagateRSJobConfiguration.java @@ -17,9 +17,7 @@ import org.springframework.batch.core.Job; import org.springframework.batch.core.Step; -import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.batch.core.job.builder.JobBuilder; -import org.springframework.batch.core.launch.support.RunIdIncrementer; import org.springframework.batch.core.repository.JobRepository; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; @@ -31,7 +29,6 @@ import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.BACK_PROPAGATE_SPLIT_OR_MERGED_RS_STEP; @Configuration -@EnableBatchProcessing public class BackPropagateRSJobConfiguration { // Deal with back-propagation of new RS, that were assigned to SS in the remapped assembly, to the original assembly @Bean(BACK_PROPAGATE_NEW_RS_JOB) @@ -40,7 +37,6 @@ public Job backPropagateNewRSJob( @Qualifier(BACK_PROPAGATE_NEW_RS_STEP) Step backPropagateNewRSStep, JobRepository jobRepository) { return new JobBuilder(BACK_PROPAGATE_NEW_RS_JOB, jobRepository) - .incrementer(new RunIdIncrementer()) .start(backPropagateNewRSStep) .build(); } @@ -53,7 +49,6 @@ public Job backPropagateSplitOrMergedRSJob( @Qualifier(BACK_PROPAGATE_SPLIT_OR_MERGED_RS_STEP) Step backPropagateSplitMergedRSStep, JobRepository jobRepository) { return new JobBuilder(BACK_PROPAGATE_SPLIT_OR_MERGED_RS_JOB, jobRepository) - .incrementer(new RunIdIncrementer()) .start(backPropagateSplitMergedRSStep) .build(); } diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/ClusterUnclusteredVariantsJobConfiguration.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/ClusterUnclusteredVariantsJobConfiguration.java index 3740f1fd2..d46e3c84d 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/ClusterUnclusteredVariantsJobConfiguration.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/ClusterUnclusteredVariantsJobConfiguration.java @@ -18,9 +18,7 @@ import org.springframework.batch.core.Job; import org.springframework.batch.core.JobExecutionListener; import org.springframework.batch.core.Step; -import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.batch.core.job.builder.JobBuilder; -import org.springframework.batch.core.launch.support.RunIdIncrementer; import org.springframework.batch.core.repository.JobRepository; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; @@ -35,7 +33,6 @@ import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.PROCESS_RS_SPLIT_CANDIDATES_STEP; @Configuration -@EnableBatchProcessing public class ClusterUnclusteredVariantsJobConfiguration { // Should be run after split or merge candidates have been processed in the step @see PROCESS_REMAPPED_VARIANTS_WITH_RS_JOB // In this step, proceed to cluster as-yet unclustered variants in a given assembly @@ -50,7 +47,6 @@ public Job clusteringFromMongoJob( @Qualifier(JOB_EXECUTION_LISTENER) JobExecutionListener jobExecutionListener, JobRepository jobRepository) { return new JobBuilder(CLUSTER_UNCLUSTERED_VARIANTS_JOB, jobRepository) - .incrementer(new RunIdIncrementer()) .start(processRSMergeCandidatesStep) .next(processRSSplitCandidatesStep) .next(clearRSMergeAndSplitCandidatesStep) diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/ClusteringFromMongoJobConfiguration.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/ClusteringFromMongoJobConfiguration.java index e4c27393d..7f9686d8b 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/ClusteringFromMongoJobConfiguration.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/ClusteringFromMongoJobConfiguration.java @@ -21,13 +21,11 @@ import org.springframework.batch.core.JobExecutionListener; import org.springframework.batch.core.Step; import org.springframework.batch.core.StepExecution; -import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.batch.core.job.builder.FlowBuilder; import org.springframework.batch.core.job.builder.JobBuilder; import org.springframework.batch.core.job.flow.FlowExecutionStatus; import org.springframework.batch.core.job.flow.JobExecutionDecider; import org.springframework.batch.core.job.flow.support.SimpleFlow; -import org.springframework.batch.core.launch.support.RunIdIncrementer; import org.springframework.batch.core.repository.JobRepository; import org.springframework.batch.core.step.builder.StepBuilder; import org.springframework.batch.repeat.RepeatStatus; @@ -48,9 +46,9 @@ import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.JOB_EXECUTION_LISTENER; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.PROCESS_RS_MERGE_CANDIDATES_STEP; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.PROCESS_RS_SPLIT_CANDIDATES_STEP; +import static uk.ac.ebi.eva.accession.core.configuration.InMemoryBatchConfiguration.BATCH_TRANSACTION_MANAGER; @Configuration -@EnableBatchProcessing public class ClusteringFromMongoJobConfiguration { private JobExecutionDecider isRemappedAssemblyPresent(InputParameters inputParameters) { return new JobExecutionDecider() { @@ -63,7 +61,8 @@ public FlowExecutionStatus decide(@NonNull JobExecution jobExecution, StepExecut }; } - private Step dummyStep(JobRepository jobRepository, PlatformTransactionManager transactionManager) { + private Step dummyStep(JobRepository jobRepository, + @Qualifier(BATCH_TRANSACTION_MANAGER) PlatformTransactionManager transactionManager) { return new StepBuilder("step_" + "dummyStep", jobRepository) .tasklet((stepContribution, chunkContext) -> RepeatStatus.FINISHED, transactionManager) @@ -83,12 +82,12 @@ public Job clusteringFromMongoJob(@Qualifier(CLUSTERING_CLUSTERED_VARIANTS_FROM_ @Qualifier(BACK_PROPAGATE_SPLIT_OR_MERGED_RS_STEP) Step backPropagateSplitMergedRSStep, @Qualifier(JOB_EXECUTION_LISTENER) JobExecutionListener jobExecutionListener, - JobRepository jobRepository, PlatformTransactionManager transactionManager, + JobRepository jobRepository, + @Qualifier(BATCH_TRANSACTION_MANAGER) PlatformTransactionManager transactionManager, InputParameters inputParameters) { JobExecutionDecider jobExecutionDecider = isRemappedAssemblyPresent(inputParameters); Step dummyStep = dummyStep(jobRepository, transactionManager); return new JobBuilder(CLUSTERING_FROM_MONGO_JOB, jobRepository) - .incrementer(new RunIdIncrementer()) //We need the dummy step here because Spring won't conditionally start the first step .start(dummyStep) .listener(jobExecutionListener) diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/ProcessRemappedVariantsWithRSJobConfiguration.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/ProcessRemappedVariantsWithRSJobConfiguration.java index 6a9092b39..a0081fb95 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/ProcessRemappedVariantsWithRSJobConfiguration.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/ProcessRemappedVariantsWithRSJobConfiguration.java @@ -17,9 +17,7 @@ import org.springframework.batch.core.Job; import org.springframework.batch.core.Step; -import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.batch.core.job.builder.JobBuilder; -import org.springframework.batch.core.launch.support.RunIdIncrementer; import org.springframework.batch.core.repository.JobRepository; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; @@ -29,7 +27,6 @@ import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.PROCESS_REMAPPED_VARIANTS_WITH_RS_JOB; @Configuration -@EnableBatchProcessing public class ProcessRemappedVariantsWithRSJobConfiguration { // Deal with remapped variants with an existing RS and create split or merge candidates as needed // Can be parallelized across multiple species @@ -38,7 +35,6 @@ public Job processRemappedVariantsWithRSJob( @Qualifier(CLUSTERING_CLUSTERED_VARIANTS_FROM_MONGO_STEP) Step clusteringClusteredVariantsFromMongoStep, JobRepository jobRepository) { return new JobBuilder(PROCESS_REMAPPED_VARIANTS_WITH_RS_JOB, jobRepository) - .incrementer(new RunIdIncrementer()) .start(clusteringClusteredVariantsFromMongoStep) .build(); } diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/RSAccessionRecoveryJobConfiguration.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/RSAccessionRecoveryJobConfiguration.java index 7070c9dbc..9755305ae 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/RSAccessionRecoveryJobConfiguration.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/RSAccessionRecoveryJobConfiguration.java @@ -3,9 +3,7 @@ import org.springframework.batch.core.Job; import org.springframework.batch.core.JobExecutionListener; import org.springframework.batch.core.Step; -import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.batch.core.job.builder.JobBuilder; -import org.springframework.batch.core.launch.support.RunIdIncrementer; import org.springframework.batch.core.repository.JobRepository; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; @@ -17,7 +15,6 @@ import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.RS_ACCESSION_RECOVERY_STEP; @Configuration -@EnableBatchProcessing public class RSAccessionRecoveryJobConfiguration { @Autowired @@ -31,7 +28,6 @@ public class RSAccessionRecoveryJobConfiguration { @Bean(RS_ACCESSION_RECOVERY_JOB) public Job createMonotonicAccessionRecoveryAgentCategoryRSJob(JobRepository jobRepository) { return new JobBuilder(RS_ACCESSION_RECOVERY_JOB, jobRepository) - .incrementer(new RunIdIncrementer()) .start(monotonicAccessionRecoveryAgentCategoryRSStep) .listener(monotonicAccessionRecoveryAgentCategoryRSJobListener) .build(); diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/StudyClusteringJobConfiguration.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/StudyClusteringJobConfiguration.java index 36344d7e6..e15dc925f 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/StudyClusteringJobConfiguration.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/StudyClusteringJobConfiguration.java @@ -18,9 +18,7 @@ import org.springframework.batch.core.Job; import org.springframework.batch.core.JobExecutionListener; import org.springframework.batch.core.Step; -import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.batch.core.job.builder.JobBuilder; -import org.springframework.batch.core.launch.support.RunIdIncrementer; import org.springframework.batch.core.repository.JobRepository; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; @@ -32,7 +30,6 @@ import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.STUDY_CLUSTERING_STEP; @Configuration -@EnableBatchProcessing public class StudyClusteringJobConfiguration { @Bean(STUDY_CLUSTERING_JOB) @@ -41,7 +38,6 @@ public Job studyClusteringJob(@Qualifier(STUDY_CLUSTERING_STEP) Step clusteringS @Qualifier(JOB_EXECUTION_LISTENER) JobExecutionListener jobExecutionListener, JobRepository jobRepository) { return new JobBuilder(STUDY_CLUSTERING_JOB, jobRepository) - .incrementer(new RunIdIncrementer()) .start(clusteringStep) .next(accessioningShutdownStep) .listener(jobExecutionListener) diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/qc/ClusteringQCJobConfiguration.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/qc/ClusteringQCJobConfiguration.java index 69717ddfd..523f0e98d 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/qc/ClusteringQCJobConfiguration.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/qc/ClusteringQCJobConfiguration.java @@ -18,9 +18,7 @@ import org.springframework.batch.core.Job; import org.springframework.batch.core.Step; import org.springframework.batch.core.StepExecutionListener; -import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.batch.core.job.builder.JobBuilder; -import org.springframework.batch.core.launch.support.RunIdIncrementer; import org.springframework.batch.core.repository.JobRepository; import org.springframework.batch.core.step.builder.StepBuilder; import org.springframework.batch.item.ItemProcessor; @@ -44,9 +42,9 @@ import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.CLUSTERED_CLUSTERING_WRITER; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.PROGRESS_LISTENER; +import static uk.ac.ebi.eva.accession.core.configuration.InMemoryBatchConfiguration.BATCH_TRANSACTION_MANAGER; @Configuration -@EnableBatchProcessing // Since this is a QC Job, the configuration is intentionally lightweight and all collected in one-place // to reduce overhead public class ClusteringQCJobConfiguration { @@ -97,7 +95,8 @@ public Step reportUnclusteredSSAndPendingMergeSplitStep( @Qualifier(PENDING_MERGE_AND_SPLIT_REPORTER) ItemWriter pendingMergeSplitReporter, @Qualifier(PROGRESS_LISTENER) StepExecutionListener progressListener, - JobRepository jobRepository, PlatformTransactionManager transactionManager, + JobRepository jobRepository, + @Qualifier(BATCH_TRANSACTION_MANAGER) PlatformTransactionManager transactionManager, SimpleCompletionPolicy chunkSizeCompletionPolicy) { return new StepBuilder(REPORT_UNCLUSTERED_SS_AND_PENDING_MERGES_AND_SPLITS_STEP, jobRepository) .chunk(chunkSizeCompletionPolicy, transactionManager) @@ -115,7 +114,8 @@ public Step reportExtraneousRSStep( @Qualifier(EXTRANEOUS_RS_REPORTER) ItemWriter extraneousRSReporter, @Qualifier(PROGRESS_LISTENER) StepExecutionListener progressListener, - JobRepository jobRepository, PlatformTransactionManager transactionManager, + JobRepository jobRepository, + @Qualifier(BATCH_TRANSACTION_MANAGER) PlatformTransactionManager transactionManager, SimpleCompletionPolicy chunkSizeCompletionPolicy) { return new StepBuilder(REPORT_EXTRANEOUS_RS_STEP, jobRepository) .chunk(chunkSizeCompletionPolicy, transactionManager) @@ -133,7 +133,6 @@ public Job ClusteringQCJob( Step reportExtraneousRSStep, JobRepository jobRepository) { return new JobBuilder(CLUSTERING_QC_JOB, jobRepository) - .incrementer(new RunIdIncrementer()) .start(reportUnclusteredSSAndPendingMergeSplitStep) .next(reportExtraneousRSStep) .build(); diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/qc/DuplicateRSAccQCJobConfiguration.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/qc/DuplicateRSAccQCJobConfiguration.java index 48294ee59..840132874 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/qc/DuplicateRSAccQCJobConfiguration.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/qc/DuplicateRSAccQCJobConfiguration.java @@ -2,9 +2,7 @@ import org.springframework.batch.core.Job; import org.springframework.batch.core.Step; -import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.batch.core.job.builder.JobBuilder; -import org.springframework.batch.core.launch.support.RunIdIncrementer; import org.springframework.batch.core.repository.JobRepository; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; @@ -15,7 +13,6 @@ import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.DUPLICATE_RS_ACC_QC_STEP; @Configuration -@EnableBatchProcessing public class DuplicateRSAccQCJobConfiguration { @Autowired @@ -25,7 +22,6 @@ public class DuplicateRSAccQCJobConfiguration { @Bean(DUPLICATE_RS_ACC_QC_JOB) public Job duplicateRSAccQCJob(JobRepository jobRepository) { return new JobBuilder(DUPLICATE_RS_ACC_QC_JOB, jobRepository) - .incrementer(new RunIdIncrementer()) .start(duplicateRSAccQCStep) .build(); } diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/qc/NewClusteredVariantsQCJobConfiguration.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/qc/NewClusteredVariantsQCJobConfiguration.java index beddc0eaa..a41f8aa8d 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/qc/NewClusteredVariantsQCJobConfiguration.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/qc/NewClusteredVariantsQCJobConfiguration.java @@ -17,9 +17,7 @@ import org.springframework.batch.core.Job; import org.springframework.batch.core.Step; -import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.batch.core.job.builder.JobBuilder; -import org.springframework.batch.core.launch.support.RunIdIncrementer; import org.springframework.batch.core.repository.JobRepository; import org.springframework.batch.core.step.builder.StepBuilder; import org.springframework.batch.item.ItemStreamReader; @@ -38,8 +36,9 @@ import java.io.File; +import static uk.ac.ebi.eva.accession.core.configuration.InMemoryBatchConfiguration.BATCH_TRANSACTION_MANAGER; + @Configuration -@EnableBatchProcessing // Since this is a QC Job, the configuration is intentionally lightweight and all collected in one-place // to reduce overhead public class NewClusteredVariantsQCJobConfiguration { @@ -80,7 +79,8 @@ public MissingCveReporter missingCveReporter(MongoTemplate mongoTemplate) { public Step reportMissingCveStep( @Qualifier(RS_REPORT_READER) ItemStreamReader rsReportReader, @Qualifier(MISSING_CVE_REPORTER) ItemWriter missingCveReporter, - JobRepository jobRepository, PlatformTransactionManager transactionManager, + JobRepository jobRepository, + @Qualifier(BATCH_TRANSACTION_MANAGER) PlatformTransactionManager transactionManager, SimpleCompletionPolicy chunkSizeCompletionPolicy) { return new StepBuilder(REPORT_MISSING_CVE_STEP, jobRepository) .chunk(chunkSizeCompletionPolicy, transactionManager) @@ -93,7 +93,6 @@ public Step reportMissingCveStep( public Job ClusteringQCJob(@Qualifier(REPORT_MISSING_CVE_STEP) Step reportMissingCveStep, JobRepository jobRepository) { return new JobBuilder(NEW_CLUSTERED_VARIANTS_QC_JOB, jobRepository) - .incrementer(new RunIdIncrementer()) .start(reportMissingCveStep) .build(); } diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/steps/AccessioningShutdownStepConfiguration.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/steps/AccessioningShutdownStepConfiguration.java index 766cf292b..27269eda8 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/steps/AccessioningShutdownStepConfiguration.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/steps/AccessioningShutdownStepConfiguration.java @@ -1,10 +1,10 @@ package uk.ac.ebi.eva.accession.clustering.configuration.batch.steps; import org.springframework.batch.core.Step; -import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.batch.core.repository.JobRepository; import org.springframework.batch.core.step.builder.StepBuilder; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.transaction.PlatformTransactionManager; @@ -12,9 +12,9 @@ import uk.ac.ebi.eva.accession.core.service.nonhuman.SubmittedVariantAccessioningService; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.ACCESSIONING_SHUTDOWN_STEP; +import static uk.ac.ebi.eva.accession.core.configuration.InMemoryBatchConfiguration.BATCH_TRANSACTION_MANAGER; @Configuration -@EnableBatchProcessing public class AccessioningShutdownStepConfiguration { @Autowired private SubmittedVariantAccessioningService submittedVariantAccessioningService; @@ -23,7 +23,8 @@ public class AccessioningShutdownStepConfiguration { private ClusteredVariantAccessioningService clusteredVariantAccessioningService; @Bean(ACCESSIONING_SHUTDOWN_STEP) - public Step accessioningShutDownStep(JobRepository jobRepository, PlatformTransactionManager transactionManager) { + public Step accessioningShutDownStep(JobRepository jobRepository, + @Qualifier(BATCH_TRANSACTION_MANAGER) PlatformTransactionManager transactionManager) { return new StepBuilder(ACCESSIONING_SHUTDOWN_STEP, jobRepository) .tasklet((contribution, chunkContext) -> { submittedVariantAccessioningService.shutDownAccessionGenerator(); diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/steps/ClusteringFromMongoStepConfiguration.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/steps/ClusteringFromMongoStepConfiguration.java index 41417be99..2fb319007 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/steps/ClusteringFromMongoStepConfiguration.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/steps/ClusteringFromMongoStepConfiguration.java @@ -18,7 +18,6 @@ import org.springframework.batch.core.Step; import org.springframework.batch.core.StepExecutionListener; -import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.batch.core.repository.JobRepository; import org.springframework.batch.core.step.builder.StepBuilder; import org.springframework.batch.core.step.tasklet.TaskletStep; @@ -61,9 +60,9 @@ import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.STUDY_CLUSTERING_STEP; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.TARGET_SS_READER_FOR_NEW_BACKPROP_RS; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.TARGET_SS_READER_FOR_SPLIT_OR_MERGED_BACKPROP_RS; +import static uk.ac.ebi.eva.accession.core.configuration.InMemoryBatchConfiguration.BATCH_TRANSACTION_MANAGER; @Configuration -@EnableBatchProcessing public class ClusteringFromMongoStepConfiguration { @Bean(CLUSTERING_CLUSTERED_VARIANTS_FROM_MONGO_STEP) @@ -72,7 +71,8 @@ public Step clusteringClusteredVariantStepMongoReader( @Qualifier(CLUSTERED_CLUSTERING_WRITER) ItemWriter submittedVariantWriter, @Qualifier(PROGRESS_LISTENER) StepExecutionListener progressListener, @Qualifier(CLUSTERED_CLUSTERING_WRITER_JOB_EXECUTION_SETTER) StepExecutionListener clusteredClusteringWriterJobExecutionSetter, - JobRepository jobRepository, PlatformTransactionManager transactionManager, + JobRepository jobRepository, + @Qualifier(BATCH_TRANSACTION_MANAGER) PlatformTransactionManager transactionManager, SimpleCompletionPolicy chunkSizeCompletionPolicy) { TaskletStep step = new StepBuilder(CLUSTERING_CLUSTERED_VARIANTS_FROM_MONGO_STEP, jobRepository) .chunk(chunkSizeCompletionPolicy, transactionManager) @@ -90,7 +90,8 @@ public Step processRSMergeCandidatesStep( ItemReader rsMergeCandidatesReader, @Qualifier(RS_MERGE_WRITER) ItemWriter rsMergeWriter, @Qualifier(PROGRESS_LISTENER) StepExecutionListener progressListener, - JobRepository jobRepository, PlatformTransactionManager transactionManager, + JobRepository jobRepository, + @Qualifier(BATCH_TRANSACTION_MANAGER) PlatformTransactionManager transactionManager, SimpleCompletionPolicy chunkSizeCompletionPolicy) { TaskletStep step = new StepBuilder(PROCESS_RS_MERGE_CANDIDATES_STEP, jobRepository) .chunk(chunkSizeCompletionPolicy, transactionManager) @@ -108,7 +109,8 @@ public Step processRSSplitCandidatesStep( @Qualifier(RS_SPLIT_WRITER) ItemWriter rsSplitWriter, @Qualifier(PROGRESS_LISTENER) StepExecutionListener progressListener, @Qualifier(RS_SPLIT_WRITER_JOB_EXECUTION_SETTER) StepExecutionListener rsSplitWriterJobExecutionSetter, - JobRepository jobRepository, PlatformTransactionManager transactionManager, + JobRepository jobRepository, + @Qualifier(BATCH_TRANSACTION_MANAGER) PlatformTransactionManager transactionManager, SimpleCompletionPolicy chunkSizeCompletionPolicy) { TaskletStep step = new StepBuilder(PROCESS_RS_SPLIT_CANDIDATES_STEP, jobRepository) .chunk(chunkSizeCompletionPolicy, transactionManager) @@ -123,7 +125,8 @@ public Step processRSSplitCandidatesStep( @Bean(CLEAR_RS_MERGE_AND_SPLIT_CANDIDATES_STEP) public Step clearRSMergeAndSplitCandidatesStep( @Qualifier(CLEAR_RS_MERGE_AND_SPLIT_CANDIDATES) ItemWriter clearRSMergeAndSplitCandidates, - JobRepository jobRepository, PlatformTransactionManager transactionManager, + JobRepository jobRepository, + @Qualifier(BATCH_TRANSACTION_MANAGER) PlatformTransactionManager transactionManager, SimpleCompletionPolicy chunkSizeCompletionPolicy) { TaskletStep step = new StepBuilder(CLEAR_RS_MERGE_AND_SPLIT_CANDIDATES_STEP, jobRepository) .chunk(chunkSizeCompletionPolicy, transactionManager) @@ -155,7 +158,8 @@ public Step clusteringNonClusteredVariantStepMongoReader( @Qualifier(NON_CLUSTERED_CLUSTERING_WRITER) ItemWriter submittedVariantWriter, @Qualifier(PROGRESS_LISTENER) StepExecutionListener progressListener, @Qualifier(NON_CLUSTERED_CLUSTERING_WRITER_JOB_EXECUTION_SETTER) StepExecutionListener nonClusteredClusteringWriterJobExecutionSetter, - JobRepository jobRepository, PlatformTransactionManager transactionManager, + JobRepository jobRepository, + @Qualifier(BATCH_TRANSACTION_MANAGER) PlatformTransactionManager transactionManager, SimpleCompletionPolicy chunkSizeCompletionPolicy) { TaskletStep step = new StepBuilder(CLUSTERING_NON_CLUSTERED_VARIANTS_FROM_MONGO_STEP, jobRepository) .chunk(chunkSizeCompletionPolicy, transactionManager) @@ -173,7 +177,8 @@ public Step backPropagateNewRSStep( ItemStreamReader backPropagatedNewRSTargetReader, @Qualifier(BACK_PROPAGATED_RS_WRITER) ItemWriter backPropagatedRSWriter, @Qualifier(PROGRESS_LISTENER) StepExecutionListener progressListener, - JobRepository jobRepository, PlatformTransactionManager transactionManager, + JobRepository jobRepository, + @Qualifier(BATCH_TRANSACTION_MANAGER) PlatformTransactionManager transactionManager, SimpleCompletionPolicy chunkSizeCompletionPolicy) { TaskletStep step = new StepBuilder(BACK_PROPAGATE_NEW_RS_STEP, jobRepository) .chunk(chunkSizeCompletionPolicy, transactionManager) @@ -190,7 +195,8 @@ public Step backPropagateSplitAndMergedRSStep( ItemStreamReader> splitOrMergedRSReader, @Qualifier(BACK_PROPAGATED_RS_WRITER) ItemWriter backPropagatedRSWriter, @Qualifier(PROGRESS_LISTENER) StepExecutionListener progressListener, - JobRepository jobRepository, PlatformTransactionManager transactionManager, + JobRepository jobRepository, + @Qualifier(BATCH_TRANSACTION_MANAGER) PlatformTransactionManager transactionManager, SimpleCompletionPolicy chunkSizeCompletionPolicy) { TaskletStep step = new StepBuilder(BACK_PROPAGATE_SPLIT_OR_MERGED_RS_STEP, jobRepository) ., List>chunk(chunkSizeCompletionPolicy, transactionManager) @@ -208,7 +214,8 @@ public Step studyClusteringStep( @Qualifier(NON_CLUSTERED_CLUSTERING_WRITER) ItemWriter submittedVariantWriter, @Qualifier(PROGRESS_LISTENER) StepExecutionListener progressListener, @Qualifier(NON_CLUSTERED_CLUSTERING_WRITER_JOB_EXECUTION_SETTER) StepExecutionListener nonClusteredClusteringWriterJobExecutionSetter, - JobRepository jobRepository, PlatformTransactionManager transactionManager, + JobRepository jobRepository, + @Qualifier(BATCH_TRANSACTION_MANAGER) PlatformTransactionManager transactionManager, SimpleCompletionPolicy chunkSizeCompletionPolicy) { TaskletStep step = new StepBuilder(STUDY_CLUSTERING_STEP, jobRepository) .chunk(chunkSizeCompletionPolicy, transactionManager) diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/steps/RSAccessionRecoveryStepConfiguration.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/steps/RSAccessionRecoveryStepConfiguration.java index f484eda83..260a8aa2e 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/steps/RSAccessionRecoveryStepConfiguration.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/steps/RSAccessionRecoveryStepConfiguration.java @@ -1,7 +1,6 @@ package uk.ac.ebi.eva.accession.clustering.configuration.batch.steps; import org.springframework.batch.core.Step; -import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.batch.core.repository.JobRepository; import org.springframework.batch.core.step.builder.StepBuilder; import org.springframework.beans.factory.annotation.Autowired; @@ -13,9 +12,9 @@ import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.RS_ACCESSION_RECOVERY_SERVICE; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.RS_ACCESSION_RECOVERY_STEP; +import static uk.ac.ebi.eva.accession.core.configuration.InMemoryBatchConfiguration.BATCH_TRANSACTION_MANAGER; @Configuration -@EnableBatchProcessing public class RSAccessionRecoveryStepConfiguration { @Autowired @Qualifier(RS_ACCESSION_RECOVERY_SERVICE) @@ -23,6 +22,7 @@ public class RSAccessionRecoveryStepConfiguration { @Bean(RS_ACCESSION_RECOVERY_STEP) public Step monotonicAccessionRecoveryAgentCategoryRSStep(JobRepository jobRepository, + @Qualifier(BATCH_TRANSACTION_MANAGER) PlatformTransactionManager transactionManager) { return new StepBuilder(RS_ACCESSION_RECOVERY_STEP, jobRepository) .tasklet((contribution, chunkContext) -> { diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/steps/qc/DuplicateRSAccQCStepConfiguration.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/steps/qc/DuplicateRSAccQCStepConfiguration.java index 2c0afce12..ca8a0a73b 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/steps/qc/DuplicateRSAccQCStepConfiguration.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/steps/qc/DuplicateRSAccQCStepConfiguration.java @@ -1,7 +1,6 @@ package uk.ac.ebi.eva.accession.clustering.configuration.batch.steps.qc; import org.springframework.batch.core.Step; -import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.batch.core.repository.JobRepository; import org.springframework.batch.core.step.builder.StepBuilder; import org.springframework.batch.core.step.tasklet.TaskletStep; @@ -21,9 +20,9 @@ import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.DUPLICATE_RS_ACC_QC_PROCESSOR; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.DUPLICATE_RS_ACC_QC_STEP; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.DUPLICATE_RS_ACC_QC_WRITER; +import static uk.ac.ebi.eva.accession.core.configuration.InMemoryBatchConfiguration.BATCH_TRANSACTION_MANAGER; @Configuration -@EnableBatchProcessing public class DuplicateRSAccQCStepConfiguration { @Autowired @@ -39,7 +38,8 @@ public class DuplicateRSAccQCStepConfiguration { private ItemWriter> duplicateRSAccQCWriter; @Bean(DUPLICATE_RS_ACC_QC_STEP) - public Step duplicateRSAccQCStep(JobRepository jobRepository, PlatformTransactionManager transactionManager) { + public Step duplicateRSAccQCStep(JobRepository jobRepository, + @Qualifier(BATCH_TRANSACTION_MANAGER) PlatformTransactionManager transactionManager) { TaskletStep step = new StepBuilder(DUPLICATE_RS_ACC_QC_STEP, jobRepository) // hardcoded the chunk size as 1, as the reader takes care of accumulating // and sending the chunk size (defined in properties file) elements to the processor diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/parameters/InputParameters.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/parameters/InputParameters.java index c2128afb8..1edf29816 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/parameters/InputParameters.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/parameters/InputParameters.java @@ -33,8 +33,6 @@ public class InputParameters { private int chunkSize; - private boolean forceRestart; - private boolean allowRetry; private String rsAccFile; @@ -73,14 +71,6 @@ public void setChunkSize(int chunkSize) { this.chunkSize = chunkSize; } - public boolean isForceRestart() { - return forceRestart; - } - - public void setForceRestart(boolean forceRestart) { - this.forceRestart = forceRestart; - } - public boolean isAllowRetry() { return allowRetry; } diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/runner/ClusteringCommandLineRunner.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/runner/ClusteringCommandLineRunner.java index 25e60ba7f..477b73613 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/runner/ClusteringCommandLineRunner.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/runner/ClusteringCommandLineRunner.java @@ -19,7 +19,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.batch.core.Job; -import org.springframework.batch.core.JobExecution; import org.springframework.batch.core.JobExecutionException; import org.springframework.batch.core.JobParameters; import org.springframework.batch.core.JobParametersInvalidException; @@ -37,10 +36,8 @@ import org.springframework.stereotype.Component; import org.springframework.util.PatternMatchUtils; import uk.ac.ebi.eva.accession.clustering.parameters.InputParameters; -import uk.ac.ebi.eva.accession.core.runner.CommandLineRunnerUtils; import uk.ac.ebi.eva.commons.batch.exception.NoJobToExecuteException; import uk.ac.ebi.eva.commons.batch.exception.NoParametersHaveBeenPassedException; -import uk.ac.ebi.eva.commons.batch.exception.NoPreviousJobExecutionException; import uk.ac.ebi.eva.commons.batch.exception.UnknownJobException; import uk.ac.ebi.eva.commons.batch.job.JobExecutionApplicationListener; import uk.ac.ebi.eva.commons.batch.job.JobStatusManager; @@ -113,17 +110,9 @@ public void run(String... args) throws JobExecutionException { JobStatusManager.checkIfJobNameHasBeenDefined(jobName); JobStatusManager.checkIfPropertiesHaveBeenProvided(jobParameters); - if (inputParameters.isForceRestart()) { - JobExecution previousJobExecution = CommandLineRunnerUtils.getLastJobExecution(jobName, jobExplorer, - jobParameters); - CommandLineRunnerUtils.markPreviousJobAsFailed(jobName, jobRepository, - previousJobExecution != null ? previousJobExecution.getJobParameters() : jobParameters); - } else { - jobParameters = CommandLineRunnerUtils.addRunIDToJobParameters(jobName, jobExplorer, jobParameters); - } launchJob(jobParameters); - } catch (NoJobToExecuteException | NoParametersHaveBeenPassedException | NoPreviousJobExecutionException - | UnknownJobException | JobParametersInvalidException | JobExecutionAlreadyRunningException e) { + } catch (NoJobToExecuteException | NoParametersHaveBeenPassedException | UnknownJobException + | JobParametersInvalidException | JobExecutionAlreadyRunningException e) { logger.error(e.getMessage()); logger.debug("Error trace", e); abnormalExit = true; diff --git a/eva-accession-clustering/src/main/resources/application.properties b/eva-accession-clustering/src/main/resources/application.properties index d63886aef..b90cd9152 100644 --- a/eva-accession-clustering/src/main/resources/application.properties +++ b/eva-accession-clustering/src/main/resources/application.properties @@ -36,12 +36,6 @@ spring.data.mongodb.password= spring.data.mongodb.authentication-database= mongodb.read-preference=secondaryPreferred -spring.datasource.driver-class-name=org.postgresql.Driver -spring.datasource.url= -spring.datasource.username= -spring.datasource.password= -spring.datasource.tomcat.max-active=3 - # Only to set up the database! # jmmut: These don't work at the moment, maybe because we use both Spring 2 and Spring 1 through variation commons # spring.jpa.generate-ddl=true diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/BackPropagatedRSWriterTest.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/BackPropagatedRSWriterTest.java index a255bf923..cd851dd37 100644 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/BackPropagatedRSWriterTest.java +++ b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/BackPropagatedRSWriterTest.java @@ -31,7 +31,7 @@ import uk.ac.ebi.ampt2d.commons.accession.core.models.AccessionWrapper; import uk.ac.ebi.ampt2d.commons.accession.hashing.SHA1HashingFunction; import uk.ac.ebi.eva.accession.clustering.test.configuration.BatchTestConfiguration; -import uk.ac.ebi.eva.accession.clustering.test.configuration.MongoTestConfiguration; +import uk.ac.ebi.eva.accession.core.configuration.ContiguousIdBlocksDataSourceConfiguration; import uk.ac.ebi.eva.accession.core.model.ClusteredVariant; import uk.ac.ebi.eva.accession.core.model.IClusteredVariant; import uk.ac.ebi.eva.accession.core.model.ISubmittedVariant; @@ -43,6 +43,7 @@ import uk.ac.ebi.eva.accession.core.service.nonhuman.SubmittedVariantAccessioningService; import uk.ac.ebi.eva.accession.core.summary.ClusteredVariantSummaryFunction; import uk.ac.ebi.eva.accession.core.summary.SubmittedVariantSummaryFunction; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.commons.core.models.VariantType; import uk.ac.ebi.eva.metrics.metric.MetricCompute; @@ -59,7 +60,8 @@ import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.CLUSTERED_CLUSTERING_WRITER; @ExtendWith(SpringExtension.class) -@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class,}) +@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class, + ContiguousIdBlocksDataSourceConfiguration.class}) @TestPropertySource("classpath:backpropagation-test.properties") public class BackPropagatedRSWriterTest extends MongoTestContainerHelper { private static final int TAXONOMY = 60711; diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/ClusteringMongoReaderTest.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/ClusteringMongoReaderTest.java index c5971393b..734e8445b 100644 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/ClusteringMongoReaderTest.java +++ b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/ClusteringMongoReaderTest.java @@ -26,9 +26,9 @@ import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit.jupiter.SpringExtension; -import uk.ac.ebi.eva.accession.clustering.test.configuration.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.MongoConfiguration; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.core.utils.MongoTestDataLoader; diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/RSMergeAndSplitCandidatesReaderTest.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/RSMergeAndSplitCandidatesReaderTest.java index 6d14878c2..2c9d8e018 100644 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/RSMergeAndSplitCandidatesReaderTest.java +++ b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/RSMergeAndSplitCandidatesReaderTest.java @@ -28,10 +28,11 @@ import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.eva.accession.clustering.configuration.batch.io.RSMergeAndSplitCandidatesReaderConfiguration; import uk.ac.ebi.eva.accession.clustering.test.configuration.BatchTestConfiguration; -import uk.ac.ebi.eva.accession.clustering.test.configuration.MongoTestConfiguration; +import uk.ac.ebi.eva.accession.core.configuration.ContiguousIdBlocksDataSourceConfiguration; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantInactiveEntity; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantOperationEntity; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.commons.mongodb.readers.MongoDbCursorItemReader; @@ -46,11 +47,9 @@ @ExtendWith(SpringExtension.class) @TestPropertySource("classpath:clustering-pipeline-test.properties") @ContextConfiguration(classes = {RSMergeAndSplitCandidatesReaderConfiguration.class, MongoTestConfiguration.class, - BatchTestConfiguration.class}) + BatchTestConfiguration.class, ContiguousIdBlocksDataSourceConfiguration.class}) public class RSMergeAndSplitCandidatesReaderTest extends MongoTestContainerHelper { - private static final String TEST_DB = "test-db"; - private static final String ASSEMBLY = "GCA_000000001.1"; private static final String SUBMITTED_VARIANT_OPERATION_COLLECTION = "submittedVariantOperationEntity"; diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/RSMergeWriterTest.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/RSMergeWriterTest.java index c18812865..77e724730 100644 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/RSMergeWriterTest.java +++ b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/RSMergeWriterTest.java @@ -40,7 +40,7 @@ import uk.ac.ebi.eva.accession.clustering.metric.ClusteringMetric; import uk.ac.ebi.eva.accession.clustering.test.DatabaseState; import uk.ac.ebi.eva.accession.clustering.test.configuration.BatchTestConfiguration; -import uk.ac.ebi.eva.accession.clustering.test.configuration.MongoTestConfiguration; +import uk.ac.ebi.eva.accession.core.configuration.ContiguousIdBlocksDataSourceConfiguration; import uk.ac.ebi.eva.accession.core.model.ClusteredVariant; import uk.ac.ebi.eva.accession.core.model.IClusteredVariant; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpClusteredVariantEntity; @@ -54,6 +54,7 @@ import uk.ac.ebi.eva.accession.core.service.nonhuman.ClusteredVariantAccessioningService; import uk.ac.ebi.eva.accession.core.service.nonhuman.SubmittedVariantAccessioningService; import uk.ac.ebi.eva.accession.core.summary.ClusteredVariantSummaryFunction; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.commons.core.models.pipeline.Variant; import uk.ac.ebi.eva.commons.mongodb.readers.MongoDbCursorItemReader; @@ -79,7 +80,7 @@ @ExtendWith(SpringExtension.class) @TestPropertySource("classpath:merge-split-test.properties") @ContextConfiguration(classes = {RSMergeAndSplitCandidatesReaderConfiguration.class, RSMergeAndSplitWriterConfiguration.class, - MongoTestConfiguration.class, BatchTestConfiguration.class}) + MongoTestConfiguration.class, BatchTestConfiguration.class, ContiguousIdBlocksDataSourceConfiguration.class}) public class RSMergeWriterTest extends MongoTestContainerHelper { private static final String ASSEMBLY = "GCA_000000001.1"; diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/RSSplitWriterTest.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/RSSplitWriterTest.java index e2c326179..9c414ba4c 100644 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/RSSplitWriterTest.java +++ b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/RSSplitWriterTest.java @@ -25,7 +25,9 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.core.io.ClassPathResource; import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.jdbc.datasource.init.ResourceDatabasePopulator; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; @@ -37,7 +39,7 @@ import uk.ac.ebi.eva.accession.clustering.metric.ClusteringMetric; import uk.ac.ebi.eva.accession.clustering.test.DatabaseState; import uk.ac.ebi.eva.accession.clustering.test.configuration.BatchTestConfiguration; -import uk.ac.ebi.eva.accession.clustering.test.configuration.MongoTestConfiguration; +import uk.ac.ebi.eva.accession.core.configuration.ContiguousIdBlocksDataSourceConfiguration; import uk.ac.ebi.eva.accession.core.model.ISubmittedVariant; import uk.ac.ebi.eva.accession.core.model.SubmittedVariant; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpClusteredVariantEntity; @@ -48,12 +50,17 @@ import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantOperationEntity; import uk.ac.ebi.eva.accession.core.service.nonhuman.SubmittedVariantAccessioningService; import uk.ac.ebi.eva.accession.core.summary.SubmittedVariantSummaryFunction; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.metrics.metric.MetricCompute; +import javax.sql.DataSource; import java.io.File; import java.io.IOException; import java.nio.file.Files; +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -70,7 +77,9 @@ @ExtendWith(SpringExtension.class) @TestPropertySource("classpath:merge-split-test.properties") @ContextConfiguration(classes = {RSMergeAndSplitCandidatesReaderConfiguration.class, - RSMergeAndSplitWriterConfiguration.class, MongoTestConfiguration.class, BatchTestConfiguration.class}) + RSMergeAndSplitWriterConfiguration.class, MongoTestConfiguration.class, BatchTestConfiguration.class, + ContiguousIdBlocksDataSourceConfiguration.class}) +@DirtiesContext(classMode = DirtiesContext.ClassMode.BEFORE_EACH_TEST_METHOD) public class RSSplitWriterTest extends MongoTestContainerHelper { private static final String ASSEMBLY = "GCA_000000001.1"; @@ -92,6 +101,9 @@ public class RSSplitWriterTest extends MongoTestContainerHelper { @MockBean private JobExecution jobExecution; + @Autowired + private DataSource dataSource; + private SubmittedVariantEntity ss1, ss2, ss3, ss4, ss5, ss6; @Autowired @@ -101,8 +113,16 @@ public class RSSplitWriterTest extends MongoTestContainerHelper { private MetricCompute metricCompute; @BeforeEach - public void setUp() throws IOException { + public void setUp() throws IOException, SQLException { mongoTemplate.getDb().drop(); + try (Connection conn = dataSource.getConnection(); + Statement stmt = conn.createStatement()) { + stmt.execute("DROP TABLE IF EXISTS contiguous_id_blocks"); + } + ResourceDatabasePopulator populator = new ResourceDatabasePopulator(); + populator.addScript(new ClassPathResource("test-data/contiguous_id_blocks_schema.sql")); + populator.execute(dataSource); + metricCompute.clearCount(); Files.deleteIfExists(this.rsReportFile.toPath()); diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/SSSplitWriterTest.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/SSSplitWriterTest.java index cbe743db0..4f036f04f 100644 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/SSSplitWriterTest.java +++ b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/SSSplitWriterTest.java @@ -25,9 +25,12 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.core.io.ClassPathResource; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.http.HttpMethod; import org.springframework.http.HttpStatus; +import org.springframework.jdbc.datasource.init.ResourceDatabasePopulator; +import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit.jupiter.SpringExtension; @@ -39,7 +42,7 @@ import uk.ac.ebi.eva.accession.clustering.metric.ClusteringMetric; import uk.ac.ebi.eva.accession.clustering.test.DatabaseState; import uk.ac.ebi.eva.accession.clustering.test.configuration.BatchTestConfiguration; -import uk.ac.ebi.eva.accession.clustering.test.configuration.MongoTestConfiguration; +import uk.ac.ebi.eva.accession.core.configuration.ContiguousIdBlocksDataSourceConfiguration; import uk.ac.ebi.eva.accession.core.model.ISubmittedVariant; import uk.ac.ebi.eva.accession.core.model.SubmittedVariant; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpSubmittedVariantEntity; @@ -48,11 +51,15 @@ import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantOperationEntity; import uk.ac.ebi.eva.accession.core.service.nonhuman.SubmittedVariantAccessioningService; import uk.ac.ebi.eva.accession.core.summary.SubmittedVariantSummaryFunction; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.metrics.count.CountServiceParameters; import uk.ac.ebi.eva.metrics.metric.MetricCompute; +import javax.sql.DataSource; import java.net.URI; +import java.sql.Connection; +import java.sql.Statement; import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -73,7 +80,8 @@ @ExtendWith(SpringExtension.class) @TestPropertySource("classpath:merge-split-test.properties") @ContextConfiguration(classes = {SSSplitWriterConfiguration.class, MongoTestConfiguration.class, - BatchTestConfiguration.class}) + BatchTestConfiguration.class, ContiguousIdBlocksDataSourceConfiguration.class}) +@DirtiesContext(classMode = DirtiesContext.ClassMode.BEFORE_EACH_TEST_METHOD) public class SSSplitWriterTest extends MongoTestContainerHelper { private static final String ASSEMBLY = "GCA_000000001.1"; @@ -101,6 +109,9 @@ public class SSSplitWriterTest extends MongoTestContainerHelper { @MockBean private JobExecution jobExecution; + @Autowired + private DataSource dataSource; + private MockRestServiceServer mockServer; @Autowired @@ -112,6 +123,15 @@ public class SSSplitWriterTest extends MongoTestContainerHelper { @BeforeEach public void setUp() throws Exception { mongoTemplate.getDb().drop(); + + try (Connection conn = dataSource.getConnection(); + Statement stmt = conn.createStatement()) { + stmt.execute("DROP TABLE IF EXISTS contiguous_id_blocks"); + } + ResourceDatabasePopulator populator = new ResourceDatabasePopulator(); + populator.addScript(new ClassPathResource("test-data/contiguous_id_blocks_schema.sql")); + populator.execute(dataSource); + metricCompute.clearCount(); mockServer = MockRestServiceServer.createServer(restTemplate); diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/StudyClusteringMongoReaderTest.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/StudyClusteringMongoReaderTest.java index 74c30f7f2..9d55e7baa 100644 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/StudyClusteringMongoReaderTest.java +++ b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/StudyClusteringMongoReaderTest.java @@ -26,9 +26,9 @@ import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit.jupiter.SpringExtension; -import uk.ac.ebi.eva.accession.clustering.test.configuration.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.MongoConfiguration; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.core.utils.MongoTestDataLoader; diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/clustering_writer/IssueAccessionClusteringWriterTest.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/clustering_writer/IssueAccessionClusteringWriterTest.java index 39f560d3b..7c72f14f9 100644 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/clustering_writer/IssueAccessionClusteringWriterTest.java +++ b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/clustering_writer/IssueAccessionClusteringWriterTest.java @@ -29,12 +29,13 @@ import org.springframework.batch.item.ItemWriter; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.core.io.ClassPathResource; import org.springframework.core.io.ResourceLoader; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.data.mongodb.core.query.Criteria; import org.springframework.data.mongodb.core.query.Query; +import org.springframework.jdbc.datasource.init.ResourceDatabasePopulator; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; @@ -45,7 +46,7 @@ import uk.ac.ebi.eva.accession.clustering.batch.io.RSSplitWriter; import uk.ac.ebi.eva.accession.clustering.parameters.InputParameters; import uk.ac.ebi.eva.accession.clustering.test.configuration.BatchTestConfiguration; -import uk.ac.ebi.eva.accession.clustering.test.configuration.MongoTestConfiguration; +import uk.ac.ebi.eva.accession.core.configuration.ContiguousIdBlocksDataSourceConfiguration; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.ClusteredVariantAccessioningConfiguration; import uk.ac.ebi.eva.accession.core.model.ClusteredVariant; import uk.ac.ebi.eva.accession.core.model.IClusteredVariant; @@ -57,15 +58,20 @@ import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantOperationEntity; import uk.ac.ebi.eva.accession.core.summary.ClusteredVariantSummaryFunction; import uk.ac.ebi.eva.accession.core.summary.SubmittedVariantSummaryFunction; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.core.utils.MongoTestDataLoader; import uk.ac.ebi.eva.commons.core.models.VariantType; import uk.ac.ebi.eva.commons.mongodb.readers.MongoDbCursorItemReader; import uk.ac.ebi.eva.metrics.metric.MetricCompute; +import javax.sql.DataSource; import java.io.File; import java.io.IOException; import java.nio.file.Files; +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -92,8 +98,8 @@ * care of those scenarios. */ @ExtendWith(SpringExtension.class) -@EnableAutoConfiguration -@ContextConfiguration(classes = {ClusteredVariantAccessioningConfiguration.class, BatchTestConfiguration.class, MongoTestConfiguration.class}) +@ContextConfiguration(classes = {ClusteredVariantAccessioningConfiguration.class, BatchTestConfiguration.class, + MongoTestConfiguration.class, ContiguousIdBlocksDataSourceConfiguration.class}) @TestPropertySource("classpath:clustering-issuance-test.properties") public class IssueAccessionClusteringWriterTest extends MongoTestContainerHelper { @@ -154,13 +160,24 @@ public class IssueAccessionClusteringWriterTest extends MongoTestContainerHelper @MockBean private JobExecution jobExecution; + @Autowired + private DataSource dataSource; + private Function hashingFunction; private Function clusteredHashingFunction; @BeforeEach - public void setUp() throws IOException { + public void setUp() throws IOException, SQLException { mongoTemplate.getDb().drop(); + try (Connection conn = dataSource.getConnection(); + Statement stmt = conn.createStatement()) { + stmt.execute("DROP TABLE IF EXISTS contiguous_id_blocks"); + } + ResourceDatabasePopulator populator = new ResourceDatabasePopulator(); + populator.addScript(new ClassPathResource("test-data/contiguous_id_blocks_schema.sql")); + populator.execute(dataSource); + hashingFunction = new SubmittedVariantSummaryFunction().andThen(new SHA1HashingFunction()); clusteredHashingFunction = new ClusteredVariantSummaryFunction().andThen(new SHA1HashingFunction()); Files.deleteIfExists(this.rsReportFile.toPath()); diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/clustering_writer/MergeAccessionClusteringWriterTest.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/clustering_writer/MergeAccessionClusteringWriterTest.java index dd014267f..2d44c6621 100644 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/clustering_writer/MergeAccessionClusteringWriterTest.java +++ b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/clustering_writer/MergeAccessionClusteringWriterTest.java @@ -25,7 +25,6 @@ import org.springframework.batch.item.ItemWriter; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.data.mongodb.core.query.Query; import org.springframework.test.annotation.DirtiesContext; @@ -38,7 +37,7 @@ import uk.ac.ebi.eva.accession.clustering.batch.io.ClusteringMongoReader; import uk.ac.ebi.eva.accession.clustering.batch.io.ClusteringWriter; import uk.ac.ebi.eva.accession.clustering.test.configuration.BatchTestConfiguration; -import uk.ac.ebi.eva.accession.clustering.test.configuration.MongoTestConfiguration; +import uk.ac.ebi.eva.accession.core.configuration.ContiguousIdBlocksDataSourceConfiguration; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.ClusteredVariantAccessioningConfiguration; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.SubmittedVariantAccessioningConfiguration; import uk.ac.ebi.eva.accession.core.model.ClusteredVariant; @@ -57,6 +56,7 @@ import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantOperationEntity; import uk.ac.ebi.eva.accession.core.summary.ClusteredVariantSummaryFunction; import uk.ac.ebi.eva.accession.core.summary.SubmittedVariantSummaryFunction; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.commons.core.models.VariantType; import uk.ac.ebi.eva.commons.mongodb.readers.MongoDbCursorItemReader; @@ -94,9 +94,9 @@ * Other test classes in this folder take care of other scenarios. */ @ExtendWith(SpringExtension.class) -@EnableAutoConfiguration @ContextConfiguration(classes = {ClusteredVariantAccessioningConfiguration.class, - SubmittedVariantAccessioningConfiguration.class, BatchTestConfiguration.class, MongoTestConfiguration.class}) + SubmittedVariantAccessioningConfiguration.class, BatchTestConfiguration.class, MongoTestConfiguration.class, + ContiguousIdBlocksDataSourceConfiguration.class}) @TestPropertySource("classpath:clustering-writer-test.properties") public class MergeAccessionClusteringWriterTest extends MongoTestContainerHelper { private static final String CLUSTERED_VARIANT_COLLECTION = "clusteredVariantEntity"; diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/clustering_writer/RemappedVariantsClusteringWriterTest.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/clustering_writer/RemappedVariantsClusteringWriterTest.java index 95be884ea..02c268677 100644 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/clustering_writer/RemappedVariantsClusteringWriterTest.java +++ b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/clustering_writer/RemappedVariantsClusteringWriterTest.java @@ -26,9 +26,10 @@ import org.springframework.batch.item.ItemWriter; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.core.io.ClassPathResource; import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.jdbc.datasource.init.ResourceDatabasePopulator; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; @@ -40,7 +41,7 @@ import uk.ac.ebi.eva.accession.clustering.batch.io.RSSplitWriter; import uk.ac.ebi.eva.accession.clustering.parameters.InputParameters; import uk.ac.ebi.eva.accession.clustering.test.configuration.BatchTestConfiguration; -import uk.ac.ebi.eva.accession.clustering.test.configuration.MongoTestConfiguration; +import uk.ac.ebi.eva.accession.core.configuration.ContiguousIdBlocksDataSourceConfiguration; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.ClusteredVariantAccessioningConfiguration; import uk.ac.ebi.eva.accession.core.model.ClusteredVariant; import uk.ac.ebi.eva.accession.core.model.IClusteredVariant; @@ -53,10 +54,15 @@ import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantOperationEntity; import uk.ac.ebi.eva.accession.core.summary.ClusteredVariantSummaryFunction; import uk.ac.ebi.eva.accession.core.summary.SubmittedVariantSummaryFunction; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.commons.core.models.VariantType; import uk.ac.ebi.eva.commons.mongodb.readers.MongoDbCursorItemReader; +import javax.sql.DataSource; +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; import java.time.LocalDateTime; import java.util.ArrayList; import java.util.Arrays; @@ -84,9 +90,8 @@ * Other test classes in this folder take care of other scenarios. */ @ExtendWith(SpringExtension.class) -@EnableAutoConfiguration @ContextConfiguration(classes = {ClusteredVariantAccessioningConfiguration.class, BatchTestConfiguration.class, - MongoTestConfiguration.class}) + MongoTestConfiguration.class, ContiguousIdBlocksDataSourceConfiguration.class}) @TestPropertySource("classpath:clustering-writer-test.properties") public class RemappedVariantsClusteringWriterTest extends MongoTestContainerHelper { private static final String CLUSTERED_VARIANT_COLLECTION = "clusteredVariantEntity"; @@ -151,15 +156,27 @@ public class RemappedVariantsClusteringWriterTest extends MongoTestContainerHelp @MockBean private JobExecution jobExecution; + @Autowired + private DataSource dataSource; + private Function hashingFunction; private Function clusteredHashingFunction; @BeforeEach - public void setUp() { + public void setUp() throws SQLException { ASM_1 = inputParameters.getRemappedFrom(); ASM_2 = inputParameters.getAssemblyAccession(); mongoTemplate.getDb().drop(); + + try (Connection conn = dataSource.getConnection(); + Statement stmt = conn.createStatement()) { + stmt.execute("DROP TABLE IF EXISTS contiguous_id_blocks"); + } + ResourceDatabasePopulator populator = new ResourceDatabasePopulator(); + populator.addScript(new ClassPathResource("test-data/contiguous_id_blocks_schema.sql")); + populator.execute(dataSource); + hashingFunction = new SubmittedVariantSummaryFunction().andThen(new SHA1HashingFunction()); clusteredHashingFunction = new ClusteredVariantSummaryFunction().andThen(new SHA1HashingFunction()); diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/clustering_writer/ReuseAccessionClusteringWriterTest.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/clustering_writer/ReuseAccessionClusteringWriterTest.java index c1b132875..b792f5b34 100644 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/clustering_writer/ReuseAccessionClusteringWriterTest.java +++ b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/clustering_writer/ReuseAccessionClusteringWriterTest.java @@ -26,11 +26,12 @@ import org.springframework.batch.item.ItemWriter; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.context.ApplicationContext; +import org.springframework.core.io.ClassPathResource; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.data.mongodb.core.query.Query; +import org.springframework.jdbc.datasource.init.ResourceDatabasePopulator; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; @@ -42,7 +43,7 @@ import uk.ac.ebi.eva.accession.clustering.metric.ClusteringMetric; import uk.ac.ebi.eva.accession.clustering.parameters.InputParameters; import uk.ac.ebi.eva.accession.clustering.test.configuration.BatchTestConfiguration; -import uk.ac.ebi.eva.accession.clustering.test.configuration.MongoTestConfiguration; +import uk.ac.ebi.eva.accession.core.configuration.ContiguousIdBlocksDataSourceConfiguration; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.ClusteredVariantAccessioningConfiguration; import uk.ac.ebi.eva.accession.core.model.ClusteredVariant; import uk.ac.ebi.eva.accession.core.model.IClusteredVariant; @@ -57,11 +58,16 @@ import uk.ac.ebi.eva.accession.core.service.nonhuman.ClusteredVariantAccessioningService; import uk.ac.ebi.eva.accession.core.summary.ClusteredVariantSummaryFunction; import uk.ac.ebi.eva.accession.core.summary.SubmittedVariantSummaryFunction; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.commons.core.models.VariantType; import uk.ac.ebi.eva.commons.mongodb.readers.MongoDbCursorItemReader; import uk.ac.ebi.eva.metrics.metric.MetricCompute; +import javax.sql.DataSource; +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -88,9 +94,8 @@ * Other test classes in this folder take care of other scenarios. */ @ExtendWith(SpringExtension.class) -@EnableAutoConfiguration @ContextConfiguration(classes = {ClusteredVariantAccessioningConfiguration.class, BatchTestConfiguration.class, - MongoTestConfiguration.class}) + MongoTestConfiguration.class, ContiguousIdBlocksDataSourceConfiguration.class}) @TestPropertySource("classpath:clustering-writer-test.properties") public class ReuseAccessionClusteringWriterTest extends MongoTestContainerHelper { private static String ASM_1; @@ -147,6 +152,10 @@ public class ReuseAccessionClusteringWriterTest extends MongoTestContainerHelper @MockBean private JobExecution jobExecution; + + @Autowired + private DataSource dataSource; + @Autowired @Qualifier(BACK_PROPAGATED_RS_WRITER) private ItemWriter backPropagatedRSWriter; @@ -156,10 +165,18 @@ public class ReuseAccessionClusteringWriterTest extends MongoTestContainerHelper private Function clusteredHashingFunction; @BeforeEach - public void setUp() { + public void setUp() throws SQLException { ASM_1 = inputParameters.getRemappedFrom(); ASM_2 = inputParameters.getAssemblyAccession(); mongoTemplate.getDb().drop(); + try (Connection conn = dataSource.getConnection(); + Statement stmt = conn.createStatement()) { + stmt.execute("DROP TABLE IF EXISTS contiguous_id_blocks"); + } + ResourceDatabasePopulator populator = new ResourceDatabasePopulator(); + populator.addScript(new ClassPathResource("test-data/contiguous_id_blocks_schema.sql")); + populator.execute(dataSource); + hashingFunction = new SubmittedVariantSummaryFunction().andThen(new SHA1HashingFunction()); clusteredHashingFunction = new ClusteredVariantSummaryFunction().andThen(new SHA1HashingFunction()); metricCompute.clearCount(); diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/ClusteringVariantJobConfigurationTest.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/ClusteringVariantJobConfigurationTest.java index f9fc8abac..016b22bb4 100644 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/ClusteringVariantJobConfigurationTest.java +++ b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/ClusteringVariantJobConfigurationTest.java @@ -25,10 +25,12 @@ import org.springframework.batch.test.JobLauncherTestUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.core.io.ClassPathResource; import org.springframework.core.io.ResourceLoader; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.http.HttpMethod; import org.springframework.http.HttpStatus; +import org.springframework.jdbc.datasource.init.ResourceDatabasePopulator; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; @@ -39,17 +41,20 @@ import uk.ac.ebi.eva.accession.clustering.configuration.batch.io.RSMergeAndSplitCandidatesReaderConfiguration; import uk.ac.ebi.eva.accession.clustering.parameters.InputParameters; import uk.ac.ebi.eva.accession.clustering.test.DatabaseState; -import uk.ac.ebi.eva.accession.clustering.test.configuration.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.accession.clustering.test.configuration.BatchTestConfiguration; -import uk.ac.ebi.eva.accession.clustering.test.configuration.MongoTestConfiguration; +import uk.ac.ebi.eva.accession.core.configuration.ContiguousIdBlocksDataSourceConfiguration; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantInactiveEntity; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantOperationEntity; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.core.utils.MongoTestDataLoader; import uk.ac.ebi.eva.metrics.count.CountServiceParameters; +import javax.sql.DataSource; import java.net.URI; +import java.sql.Connection; +import java.sql.Statement; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -78,7 +83,7 @@ @ExtendWith(SpringExtension.class) @ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class, - BatchJobRepositoryTestConfiguration.class}) + ContiguousIdBlocksDataSourceConfiguration.class}) @TestPropertySource("classpath:clustering-writer-test.properties") public class ClusteringVariantJobConfigurationTest extends MongoTestContainerHelper { @Autowired @@ -107,6 +112,9 @@ public class ClusteringVariantJobConfigurationTest extends MongoTestContainerHel @Autowired private CountServiceParameters countServiceParameters; + @Autowired + private DataSource dataSource; + @Autowired @Qualifier("COUNT_STATS_REST_TEMPLATE") private RestTemplate restTemplate; @@ -120,6 +128,14 @@ public void init() throws Exception { mockServer.expect(ExpectedCount.manyTimes(), requestTo(new URI(countServiceParameters.getUrl() + URL_PATH_SAVE_COUNT))) .andExpect(method(HttpMethod.POST)) .andRespond(withStatus(HttpStatus.OK)); + + try (Connection conn = dataSource.getConnection(); + Statement stmt = conn.createStatement()) { + stmt.execute("DROP TABLE IF EXISTS contiguous_id_blocks"); + } + ResourceDatabasePopulator populator = new ResourceDatabasePopulator(); + populator.addScript(new ClassPathResource("test-data/contiguous_id_blocks_schema.sql")); + populator.execute(dataSource); } @AfterEach diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/RSAccessionRecoveryTest.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/RSAccessionRecoveryTest.java index 7df9926c5..a976f125d 100644 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/RSAccessionRecoveryTest.java +++ b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/RSAccessionRecoveryTest.java @@ -26,22 +26,29 @@ import org.springframework.batch.test.context.SpringBatchTest; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.core.io.ClassPathResource; import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.jdbc.datasource.init.ResourceDatabasePopulator; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.ampt2d.commons.accession.persistence.jpa.monotonic.entities.ContiguousIdBlock; import uk.ac.ebi.ampt2d.commons.accession.persistence.jpa.monotonic.repositories.ContiguousIdBlockRepository; -import uk.ac.ebi.eva.accession.clustering.test.configuration.BatchJobRepositoryTestConfiguration; -import uk.ac.ebi.eva.accession.clustering.test.configuration.MongoTestConfiguration; import uk.ac.ebi.eva.accession.clustering.test.configuration.RSAccessionRecoveryTestConfiguration; +import uk.ac.ebi.eva.accession.core.configuration.ContiguousIdBlocksDataSourceConfiguration; +import uk.ac.ebi.eva.accession.core.configuration.InMemoryBatchConfiguration; import uk.ac.ebi.eva.accession.core.model.ClusteredVariant; import uk.ac.ebi.eva.accession.core.model.eva.ClusteredVariantEntity; import uk.ac.ebi.eva.accession.core.repository.nonhuman.eva.ClusteredVariantAccessioningRepository; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.commons.core.models.VariantType; +import javax.sql.DataSource; +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; import java.time.LocalDateTime; import java.util.ArrayList; import java.util.List; @@ -51,7 +58,8 @@ import static uk.ac.ebi.eva.accession.clustering.test.configuration.RSAccessionRecoveryTestConfiguration.JOB_LAUNCHER_RS_ACCESSION_RECOVERY; @ExtendWith(SpringExtension.class) -@ContextConfiguration(classes = {RSAccessionRecoveryTestConfiguration.class, MongoTestConfiguration.class, BatchJobRepositoryTestConfiguration.class}) +@ContextConfiguration(classes = {RSAccessionRecoveryTestConfiguration.class, MongoTestConfiguration.class, + ContiguousIdBlocksDataSourceConfiguration.class, InMemoryBatchConfiguration.class}) @TestPropertySource("classpath:rs-accession-recovery.properties") @SpringBatchTest public class RSAccessionRecoveryTest extends MongoTestContainerHelper { @@ -64,9 +72,20 @@ public class RSAccessionRecoveryTest extends MongoTestContainerHelper { @Autowired private MongoTemplate mongoTemplate; + @Autowired + private DataSource dataSource; + @BeforeEach - public void setUp() { + public void setUp() throws SQLException { mongoTemplate.getDb().drop(); + try (Connection conn = dataSource.getConnection(); + Statement stmt = conn.createStatement()) { + stmt.execute("DROP TABLE IF EXISTS contiguous_id_blocks"); + } + ResourceDatabasePopulator populator = new ResourceDatabasePopulator(); + populator.addScript(new ClassPathResource("test-data/contiguous_id_blocks_schema.sql")); + populator.addScript(new ClassPathResource("test-data/rs_accession_recovery_test_data.sql")); + populator.execute(dataSource); } @AfterEach @@ -203,7 +222,7 @@ private void verifyEndDBState() { assertEquals(3000000000l, block1.getFirstValue()); assertEquals(3000000029l, block1.getLastCommitted()); assertEquals(3000000029l, block1.getLastValue()); - assertEquals("0", block1.getApplicationInstanceId()); + assertEquals("1", block1.getApplicationInstanceId()); assertTrue(block1.isNotReserved()); // Block Recovered partially - (used 3000000030-3000000034 and 3000000040-3000000059) @@ -212,7 +231,7 @@ private void verifyEndDBState() { assertEquals(3000000030l, block2.getFirstValue()); assertEquals(3000000034l, block2.getLastCommitted()); assertEquals(3000000059l, block2.getLastValue()); - assertEquals("0", block2.getApplicationInstanceId()); + assertEquals("1", block2.getApplicationInstanceId()); assertTrue(block2.isNotReserved()); // Block Recovered - recovered entire block @@ -220,7 +239,7 @@ private void verifyEndDBState() { assertEquals(3000000060l, block3.getFirstValue()); assertEquals(3000000089l, block3.getLastCommitted()); assertEquals(3000000089l, block3.getLastValue()); - assertEquals("0", block3.getApplicationInstanceId()); + assertEquals("1", block3.getApplicationInstanceId()); assertTrue(block3.isNotReserved()); // Block Recovered - None of the accessions are used, just released the block @@ -228,7 +247,7 @@ private void verifyEndDBState() { assertEquals(3000000090l, block4.getFirstValue()); assertEquals(3000000089l, block4.getLastCommitted()); assertEquals(3000000119l, block4.getLastValue()); - assertEquals("0", block4.getApplicationInstanceId()); + assertEquals("1", block4.getApplicationInstanceId()); assertTrue(block4.isNotReserved()); // Block Not Recovered - Block not to be recovered as it's last update is after cut off time diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/qc/DuplicateRSAccQCStepConfigurationTest.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/qc/DuplicateRSAccQCStepConfigurationTest.java index d3b308691..fda9068d7 100644 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/qc/DuplicateRSAccQCStepConfigurationTest.java +++ b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/qc/DuplicateRSAccQCStepConfigurationTest.java @@ -30,15 +30,16 @@ import org.springframework.batch.test.JobLauncherTestUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.core.io.ClassPathResource; import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.jdbc.datasource.init.ResourceDatabasePopulator; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.ampt2d.commons.accession.hashing.SHA1HashingFunction; import uk.ac.ebi.eva.accession.clustering.batch.io.qc.DuplicateRSAccQCResult; -import uk.ac.ebi.eva.accession.clustering.test.configuration.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.accession.clustering.test.configuration.BatchTestConfiguration; -import uk.ac.ebi.eva.accession.clustering.test.configuration.MongoTestConfiguration; +import uk.ac.ebi.eva.accession.core.configuration.ContiguousIdBlocksDataSourceConfiguration; import uk.ac.ebi.eva.accession.core.model.ClusteredVariant; import uk.ac.ebi.eva.accession.core.model.IClusteredVariant; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpClusteredVariantEntity; @@ -46,12 +47,17 @@ import uk.ac.ebi.eva.accession.core.model.eva.ClusteredVariantEntity; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; import uk.ac.ebi.eva.accession.core.summary.ClusteredVariantSummaryFunction; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.commons.core.models.pipeline.Variant; +import javax.sql.DataSource; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Paths; +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; import java.util.Arrays; import java.util.HashSet; import java.util.List; @@ -66,7 +72,8 @@ import static uk.ac.ebi.eva.accession.clustering.test.configuration.BatchTestConfiguration.JOB_LAUNCHER_DUPLICATE_RS_ACC_QC_JOB; @ExtendWith(SpringExtension.class) -@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class, BatchJobRepositoryTestConfiguration.class}) +@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class, + ContiguousIdBlocksDataSourceConfiguration.class}) @TestPropertySource("classpath:duplicate-rs-acc-qc-test.properties") public class DuplicateRSAccQCStepConfigurationTest extends MongoTestContainerHelper { private static final String duplicateRsAccFile = "src/test/resources/duplicateRSAcc.csv"; @@ -78,12 +85,23 @@ public class DuplicateRSAccQCStepConfigurationTest extends MongoTestContainerHel @Autowired private MongoTemplate mongoTemplate; + @Autowired + private DataSource dataSource; + private final Gson gson = new GsonBuilder().create(); @BeforeEach - public void setUp() throws IOException { + public void setUp() throws IOException, SQLException { mongoTemplate.getDb().drop(); Files.deleteIfExists(Paths.get(duplicateRsAccFile)); + + try (Connection conn = dataSource.getConnection(); + Statement stmt = conn.createStatement()) { + stmt.execute("DROP TABLE IF EXISTS contiguous_id_blocks"); + } + ResourceDatabasePopulator populator = new ResourceDatabasePopulator(); + populator.addScript(new ClassPathResource("test-data/contiguous_id_blocks_schema.sql")); + populator.execute(dataSource); } @AfterEach diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/qc/NewClusteredVariantsQCJobConfigurationTest.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/qc/NewClusteredVariantsQCJobConfigurationTest.java index 91b020eb3..c5cdbd8a7 100644 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/qc/NewClusteredVariantsQCJobConfigurationTest.java +++ b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/qc/NewClusteredVariantsQCJobConfigurationTest.java @@ -30,9 +30,9 @@ import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit.jupiter.SpringExtension; -import uk.ac.ebi.eva.accession.clustering.test.configuration.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.accession.clustering.test.configuration.BatchTestConfiguration; -import uk.ac.ebi.eva.accession.clustering.test.configuration.MongoTestConfiguration; +import uk.ac.ebi.eva.accession.core.configuration.ContiguousIdBlocksDataSourceConfiguration; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.core.utils.MongoTestDataLoader; @@ -46,7 +46,8 @@ import static uk.ac.ebi.eva.accession.clustering.test.configuration.BatchTestConfiguration.JOB_LAUNCHER_NEW_CLUSTERED_VARIANTS_QC; @ExtendWith(SpringExtension.class) -@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class, BatchJobRepositoryTestConfiguration.class}) +@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class, + ContiguousIdBlocksDataSourceConfiguration.class}) @TestPropertySource("classpath:clustering-qc-test.properties") public class NewClusteredVariantsQCJobConfigurationTest extends MongoTestContainerHelper { diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/steps/ClusteringVariantStepConfigurationTest.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/steps/ClusteringVariantStepConfigurationTest.java index 2aa9ddbe5..4fcf32e3f 100644 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/steps/ClusteringVariantStepConfigurationTest.java +++ b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/steps/ClusteringVariantStepConfigurationTest.java @@ -28,10 +28,12 @@ import org.springframework.batch.test.JobLauncherTestUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.core.io.ClassPathResource; import org.springframework.core.io.ResourceLoader; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.http.HttpMethod; import org.springframework.http.HttpStatus; +import org.springframework.jdbc.datasource.init.ResourceDatabasePopulator; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; @@ -40,15 +42,18 @@ import org.springframework.test.web.client.MockRestServiceServer; import org.springframework.web.client.RestTemplate; import uk.ac.ebi.ampt2d.commons.accession.core.models.EventType; -import uk.ac.ebi.eva.accession.clustering.test.configuration.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.accession.clustering.test.configuration.BatchTestConfiguration; -import uk.ac.ebi.eva.accession.clustering.test.configuration.MongoTestConfiguration; +import uk.ac.ebi.eva.accession.core.configuration.ContiguousIdBlocksDataSourceConfiguration; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantOperationEntity; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.core.utils.MongoTestDataLoader; import uk.ac.ebi.eva.metrics.count.CountServiceParameters; +import javax.sql.DataSource; import java.net.URI; +import java.sql.Connection; +import java.sql.Statement; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -68,7 +73,7 @@ @ExtendWith(SpringExtension.class) @ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class, - BatchJobRepositoryTestConfiguration.class}) + ContiguousIdBlocksDataSourceConfiguration.class}) @TestPropertySource("classpath:clustering-issuance-test.properties") public class ClusteringVariantStepConfigurationTest extends MongoTestContainerHelper { private static final String CLUSTERED_VARIANT_COLLECTION = "clusteredVariantEntity"; @@ -90,6 +95,9 @@ public class ClusteringVariantStepConfigurationTest extends MongoTestContainerHe @Autowired private MongoTemplate mongoTemplate; + @Autowired + private DataSource dataSource; + @Autowired private ResourceLoader resourceLoader; @@ -112,6 +120,14 @@ public void init() throws Exception { mockServer.expect(ExpectedCount.manyTimes(), requestTo(new URI(countServiceParameters.getUrl() + URL_PATH_SAVE_COUNT))) .andExpect(method(HttpMethod.POST)) .andRespond(withStatus(HttpStatus.OK)); + + try (Connection conn = dataSource.getConnection(); + Statement stmt = conn.createStatement()) { + stmt.execute("DROP TABLE IF EXISTS contiguous_id_blocks"); + } + ResourceDatabasePopulator populator = new ResourceDatabasePopulator(); + populator.addScript(new ClassPathResource("test-data/contiguous_id_blocks_schema.sql")); + populator.execute(dataSource); } @AfterEach diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/runner/ClusteringCommandLineRunnerTest.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/runner/ClusteringCommandLineRunnerTest.java index 7206b6cf1..97cc522a2 100644 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/runner/ClusteringCommandLineRunnerTest.java +++ b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/runner/ClusteringCommandLineRunnerTest.java @@ -25,7 +25,6 @@ import org.mockito.Mockito; import org.springframework.batch.core.JobExecution; import org.springframework.batch.core.JobExecutionException; -import org.springframework.batch.core.JobInstance; import org.springframework.batch.core.explore.JobExplorer; import org.springframework.batch.core.repository.JobRepository; import org.springframework.batch.item.Chunk; @@ -36,11 +35,13 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.core.io.ClassPathResource; import org.springframework.core.io.ResourceLoader; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.data.mongodb.core.query.Query; import org.springframework.http.HttpMethod; import org.springframework.http.HttpStatus; +import org.springframework.jdbc.datasource.init.ResourceDatabasePopulator; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; @@ -62,9 +63,8 @@ import uk.ac.ebi.eva.accession.clustering.configuration.batch.io.RSMergeAndSplitCandidatesReaderConfiguration; import uk.ac.ebi.eva.accession.clustering.parameters.InputParameters; import uk.ac.ebi.eva.accession.clustering.test.DatabaseState; -import uk.ac.ebi.eva.accession.clustering.test.configuration.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.accession.clustering.test.configuration.BatchTestConfiguration; -import uk.ac.ebi.eva.accession.clustering.test.configuration.MongoTestConfiguration; +import uk.ac.ebi.eva.accession.core.configuration.ContiguousIdBlocksDataSourceConfiguration; import uk.ac.ebi.eva.accession.core.model.ClusteredVariant; import uk.ac.ebi.eva.accession.core.model.IClusteredVariant; import uk.ac.ebi.eva.accession.core.model.ISubmittedVariant; @@ -77,11 +77,11 @@ import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantInactiveEntity; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantOperationEntity; -import uk.ac.ebi.eva.accession.core.runner.CommandLineRunnerUtils; import uk.ac.ebi.eva.accession.core.service.nonhuman.ClusteredVariantAccessioningService; import uk.ac.ebi.eva.accession.core.service.nonhuman.SubmittedVariantAccessioningService; import uk.ac.ebi.eva.accession.core.summary.ClusteredVariantSummaryFunction; import uk.ac.ebi.eva.accession.core.summary.SubmittedVariantSummaryFunction; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.core.utils.MongoTestDataLoader; import uk.ac.ebi.eva.commons.core.models.VariantClassifier; @@ -89,8 +89,11 @@ import uk.ac.ebi.eva.commons.mongodb.readers.MongoDbCursorItemReader; import uk.ac.ebi.eva.metrics.count.CountServiceParameters; +import javax.sql.DataSource; import java.io.File; import java.net.URI; +import java.sql.Connection; +import java.sql.Statement; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -108,24 +111,17 @@ import static org.springframework.test.web.client.match.MockRestRequestMatchers.method; import static org.springframework.test.web.client.match.MockRestRequestMatchers.requestTo; import static org.springframework.test.web.client.response.MockRestResponseCreators.withStatus; -import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.BACK_PROPAGATE_NEW_RS_STEP; -import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.BACK_PROPAGATE_SPLIT_OR_MERGED_RS_STEP; -import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.CLEAR_RS_MERGE_AND_SPLIT_CANDIDATES_STEP; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.CLUSTERED_CLUSTERING_WRITER; -import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.CLUSTERING_CLUSTERED_VARIANTS_FROM_MONGO_STEP; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.CLUSTERING_FROM_MONGO_JOB; -import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.CLUSTERING_NON_CLUSTERED_VARIANTS_FROM_MONGO_STEP; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.CLUSTER_UNCLUSTERED_VARIANTS_JOB; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.PROCESS_REMAPPED_VARIANTS_WITH_RS_JOB; -import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.PROCESS_RS_MERGE_CANDIDATES_STEP; -import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.PROCESS_RS_SPLIT_CANDIDATES_STEP; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.RS_MERGE_WRITER; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.RS_SPLIT_WRITER; import static uk.ac.ebi.eva.accession.clustering.test.configuration.BatchTestConfiguration.JOB_LAUNCHER_FROM_MONGO_ONLY_FIRST_STEP; @ExtendWith(SpringExtension.class) @ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class, - BatchJobRepositoryTestConfiguration.class}) + ContiguousIdBlocksDataSourceConfiguration.class}) @TestPropertySource("classpath:clustering-pipeline-test.properties") public class ClusteringCommandLineRunnerTest extends MongoTestContainerHelper { private static String TEST_APPLICATION_INSTANCE_ID = "test-application-instance-id"; @@ -203,6 +199,9 @@ public class ClusteringCommandLineRunnerTest extends MongoTestContainerHelper { @Qualifier(RS_SPLIT_WRITER) private RSSplitWriter rsSplitWriter; + @Autowired + private DataSource dataSource; + @MockBean private JobExecution jobExecution; @@ -274,7 +273,6 @@ public void setUp() throws Exception { ASM2 = inputParameters.getAssemblyAccession(); jobRepositoryTestUtils = new JobRepositoryTestUtils(jobRepository); jobRepositoryTestUtils.removeJobExecutions(); - inputParameters.setForceRestart(false); inputParameters.setRemappedFrom(originalRemappedFrom); mockServer = MockRestServiceServer.createServer(restTemplate); @@ -282,6 +280,14 @@ public void setUp() throws Exception { .andExpect(method(HttpMethod.POST)) .andRespond(withStatus(HttpStatus.OK)); mongoTemplate.getDb().drop(); + try (Connection conn = dataSource.getConnection(); + Statement stmt = conn.createStatement()) { + stmt.execute("DROP TABLE IF EXISTS contiguous_id_blocks"); + } + ResourceDatabasePopulator populator = new ResourceDatabasePopulator(); + populator.addScript(new ClassPathResource("test-data/contiguous_id_blocks_schema.sql")); + populator.execute(dataSource); + Mockito.when(jobExecution.getJobId()).thenReturn(1L); rsSplitWriter.setJobExecution(jobExecution); @@ -290,7 +296,6 @@ public void setUp() throws Exception { @AfterEach public void tearDown() { jobRepositoryTestUtils.removeJobExecutions(); - inputParameters.setForceRestart(false); mongoTemplate.getDb().drop(); } @@ -302,25 +307,6 @@ public void runFullClusteringFromMongoJobWithNoErrors() throws JobExecutionExcep runner.setJobName(CLUSTERING_FROM_MONGO_JOB); runner.run(); assertEquals(ClusteringCommandLineRunner.EXIT_WITHOUT_ERRORS, runner.getExitCode()); - - JobInstance currentJobInstance = - CommandLineRunnerUtils.getLastJobExecution(CLUSTERING_FROM_MONGO_JOB, - jobExplorer, inputParameters.toJobParameters()) - .getJobInstance(); - assertEquals(1, jobRepository.getStepExecutionCount(currentJobInstance, - CLUSTERING_CLUSTERED_VARIANTS_FROM_MONGO_STEP)); - assertEquals(1, jobRepository.getStepExecutionCount(currentJobInstance, - PROCESS_RS_MERGE_CANDIDATES_STEP)); - assertEquals(1, jobRepository.getStepExecutionCount(currentJobInstance, - PROCESS_RS_SPLIT_CANDIDATES_STEP)); - assertEquals(1, jobRepository.getStepExecutionCount(currentJobInstance, - CLEAR_RS_MERGE_AND_SPLIT_CANDIDATES_STEP)); - assertEquals(1, jobRepository.getStepExecutionCount(currentJobInstance, - CLUSTERING_NON_CLUSTERED_VARIANTS_FROM_MONGO_STEP)); - assertEquals(1, jobRepository.getStepExecutionCount(currentJobInstance, - BACK_PROPAGATE_NEW_RS_STEP)); - assertEquals(1, jobRepository.getStepExecutionCount(currentJobInstance, - BACK_PROPAGATE_SPLIT_OR_MERGED_RS_STEP)); } @Test @@ -334,24 +320,6 @@ public void runPartialClusteringFromMongoJobForNonRemappedVariants() throws JobE runner.setJobName(CLUSTERING_FROM_MONGO_JOB); runner.run(); assertEquals(ClusteringCommandLineRunner.EXIT_WITHOUT_ERRORS, runner.getExitCode()); - - JobInstance currentJobInstance = - CommandLineRunnerUtils.getLastJobExecution(CLUSTERING_FROM_MONGO_JOB, - jobExplorer, inputParameters.toJobParameters()).getJobInstance(); - assertEquals(0, jobRepository.getStepExecutionCount(currentJobInstance, - CLUSTERING_CLUSTERED_VARIANTS_FROM_MONGO_STEP)); - assertEquals(0, jobRepository.getStepExecutionCount(currentJobInstance, - PROCESS_RS_MERGE_CANDIDATES_STEP)); - assertEquals(0, jobRepository.getStepExecutionCount(currentJobInstance, - PROCESS_RS_SPLIT_CANDIDATES_STEP)); - assertEquals(0, jobRepository.getStepExecutionCount(currentJobInstance, - CLEAR_RS_MERGE_AND_SPLIT_CANDIDATES_STEP)); - assertEquals(1, jobRepository.getStepExecutionCount(currentJobInstance, - CLUSTERING_NON_CLUSTERED_VARIANTS_FROM_MONGO_STEP)); - assertEquals(0, jobRepository.getStepExecutionCount(currentJobInstance, - BACK_PROPAGATE_NEW_RS_STEP)); - assertEquals(0, jobRepository.getStepExecutionCount(currentJobInstance, - BACK_PROPAGATE_SPLIT_OR_MERGED_RS_STEP)); } @Test @@ -920,30 +888,6 @@ public void runNonExistentJob() throws JobExecutionException { assertEquals(ClusteringCommandLineRunner.EXIT_WITH_ERRORS, runner.getExitCode()); } - @Test - @DirtiesContext - public void restartCompletedJobThatIsAlreadyInTheRepository() throws Exception { - runner.setJobName(CLUSTERING_FROM_MONGO_JOB); - runner.run(); - assertEquals(ClusteringCommandLineRunner.EXIT_WITHOUT_ERRORS, runner.getExitCode()); - - inputParameters.setForceRestart(true); - runner.run(); - assertEquals(ClusteringCommandLineRunner.EXIT_WITHOUT_ERRORS, runner.getExitCode()); - } - - - @Test - @DirtiesContext - public void forceRestartButNoJobInTheRepository() throws Exception { - runner.setJobName(CLUSTERING_FROM_MONGO_JOB); - inputParameters.setForceRestart(true); - assertEquals(Collections.EMPTY_LIST, jobExplorer.getJobNames()); - runner.run(); - - assertEquals(ClusteringCommandLineRunner.EXIT_WITHOUT_ERRORS, runner.getExitCode()); - } - @Test @DirtiesContext // See https://docs.google.com/spreadsheets/d/1KQLVCUy-vqXKgkCDt2czX6kuMfsjfCc9uBsS19MZ6dY/edit#rangeid=972582890 diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/test/configuration/BatchJobRepositoryTestConfiguration.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/test/configuration/BatchJobRepositoryTestConfiguration.java deleted file mode 100644 index 6cebe2481..000000000 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/test/configuration/BatchJobRepositoryTestConfiguration.java +++ /dev/null @@ -1,22 +0,0 @@ -package uk.ac.ebi.eva.accession.clustering.test.configuration; - -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Primary; -import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseBuilder; -import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseType; - -import javax.sql.DataSource; - -@Configuration -public class BatchJobRepositoryTestConfiguration { - - @Bean - @Primary - public DataSource dataSource() { - return new EmbeddedDatabaseBuilder() - .setType(EmbeddedDatabaseType.HSQL) - .addScript("org/springframework/batch/core/schema-hsqldb.sql") - .build(); - } -} \ No newline at end of file diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/test/configuration/BatchTestConfiguration.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/test/configuration/BatchTestConfiguration.java index cc5eb69f8..40f129633 100644 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/test/configuration/BatchTestConfiguration.java +++ b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/test/configuration/BatchTestConfiguration.java @@ -23,10 +23,10 @@ import org.springframework.batch.test.JobLauncherTestUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Import; import org.springframework.core.io.ResourceLoader; +import org.springframework.web.client.RestTemplate; import uk.ac.ebi.eva.accession.clustering.configuration.batch.io.BackPropagatedRSWriterConfiguration; import uk.ac.ebi.eva.accession.clustering.configuration.batch.io.ClusteringMongoReaderConfiguration; import uk.ac.ebi.eva.accession.clustering.configuration.batch.io.ClusteringWriterConfiguration; @@ -55,6 +55,7 @@ import uk.ac.ebi.eva.accession.clustering.configuration.batch.steps.RSAccessionRecoveryStepConfiguration; import uk.ac.ebi.eva.accession.clustering.configuration.batch.steps.qc.DuplicateRSAccQCStepConfiguration; import uk.ac.ebi.eva.accession.clustering.runner.ClusteringCommandLineRunner; +import uk.ac.ebi.eva.accession.core.configuration.InMemoryBatchConfiguration; import uk.ac.ebi.eva.commons.batch.job.JobExecutionApplicationListener; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.CLUSTERING_FROM_MONGO_JOB; @@ -64,8 +65,8 @@ import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.STUDY_CLUSTERING_JOB; import static uk.ac.ebi.eva.accession.clustering.configuration.batch.jobs.qc.NewClusteredVariantsQCJobConfiguration.NEW_CLUSTERED_VARIANTS_QC_JOB; -@EnableAutoConfiguration -@Import({ClusteringFromMongoJobConfiguration.class, +@Import({InMemoryBatchConfiguration.class, + ClusteringFromMongoJobConfiguration.class, StudyClusteringJobConfiguration.class, NewClusteredVariantsQCJobConfiguration.class, ProcessRemappedVariantsWithRSJobConfiguration.class, @@ -182,4 +183,10 @@ public JobLauncherTestUtils jobLauncherTestUtilsDuplicateRSAccQCJob(JobLauncher utils.setJob(job); return utils; } + + @Bean(name = "COUNT_STATS_REST_TEMPLATE") + public RestTemplate countStatsRestTemplate() { + return new RestTemplate(); + } + } diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/test/configuration/MongoTestConfiguration.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/test/configuration/MongoTestConfiguration.java deleted file mode 100644 index a4a8fc2c0..000000000 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/test/configuration/MongoTestConfiguration.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Copyright 2020 EMBL - European Bioinformatics Institute - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package uk.ac.ebi.eva.accession.clustering.test.configuration; - -import com.mongodb.ReadPreference; -import com.mongodb.WriteConcern; -import com.mongodb.client.MongoClient; -import com.mongodb.client.MongoClients; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.domain.EntityScan; -import org.springframework.boot.test.autoconfigure.data.mongo.AutoConfigureDataMongo; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Primary; -import org.springframework.data.mongodb.MongoDatabaseFactory; -import org.springframework.data.mongodb.config.EnableMongoAuditing; -import org.springframework.data.mongodb.core.MongoTemplate; -import org.springframework.data.mongodb.core.SimpleMongoClientDatabaseFactory; -import org.springframework.data.mongodb.core.WriteResultChecking; -import org.springframework.data.mongodb.core.convert.DefaultDbRefResolver; -import org.springframework.data.mongodb.core.convert.DefaultMongoTypeMapper; -import org.springframework.data.mongodb.core.convert.MappingMongoConverter; -import org.springframework.data.mongodb.core.convert.MongoCustomConversions; -import org.springframework.data.mongodb.core.mapping.MongoMappingContext; -import org.springframework.data.mongodb.repository.config.EnableMongoRepositories; - -import java.util.Collections; - -@Configuration -@EntityScan(basePackages = {"uk.ac.ebi.eva.accession.core.persistence"}) -@EnableMongoRepositories(basePackages = { - "uk.ac.ebi.eva.accession.core.persistence", - "uk.ac.ebi.ampt2d.commons.accession.persistence.mongodb.repository"}) -@EnableMongoAuditing -@AutoConfigureDataMongo -public class MongoTestConfiguration { - - @Value("${spring.data.mongodb.uri}") - private String mongoUri; - - @Value("${spring.data.mongodb.database}") - private String database; - - @Value("${mongodb.read-preference:primary}") - private String readPreference; - - @Bean - @Primary - public MongoClient mongoClient() { - return MongoClients.create(mongoUri); - } - - @Bean("primaryFactory") - @Primary - public MongoDatabaseFactory mongoDbFactory() { - return new SimpleMongoClientDatabaseFactory(mongoClient(), database); - } - - @Bean - @Primary - public MongoCustomConversions mongoCustomConversions() { - return new MongoCustomConversions(Collections.emptyList()); - } - - @Bean - @Primary - public MongoMappingContext mongoMappingContext(MongoCustomConversions conversions) { - MongoMappingContext context = new MongoMappingContext(); - context.setSimpleTypeHolder(conversions.getSimpleTypeHolder()); - context.setAutoIndexCreation(false); - return context; - } - - @Bean - @Primary - public MongoTemplate mongoTemplate(MongoMappingContext mongoMappingContext, - MongoCustomConversions conversions) { - MappingMongoConverter converter = new MappingMongoConverter( - new DefaultDbRefResolver(mongoDbFactory()), mongoMappingContext); - converter.setCustomConversions(conversions); - converter.setTypeMapper(new DefaultMongoTypeMapper(null)); - converter.afterPropertiesSet(); - MongoTemplate mongoTemplate = new MongoTemplate(mongoDbFactory(), converter); - mongoTemplate.setReadPreference(ReadPreference.valueOf(readPreference)); - mongoTemplate.setWriteConcern(WriteConcern.MAJORITY); - mongoTemplate.setWriteResultChecking(WriteResultChecking.EXCEPTION); - return mongoTemplate; - } -} \ No newline at end of file diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/test/configuration/RSAccessionRecoveryTestConfiguration.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/test/configuration/RSAccessionRecoveryTestConfiguration.java index 6cd5d315c..0c73614a9 100644 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/test/configuration/RSAccessionRecoveryTestConfiguration.java +++ b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/test/configuration/RSAccessionRecoveryTestConfiguration.java @@ -21,7 +21,6 @@ import org.springframework.batch.core.repository.JobRepository; import org.springframework.batch.test.JobLauncherTestUtils; import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Import; import uk.ac.ebi.eva.accession.clustering.configuration.batch.jobs.RSAccessionRecoveryJobConfiguration; @@ -32,7 +31,6 @@ import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.RS_ACCESSION_RECOVERY_JOB; -@EnableAutoConfiguration @Import({RSAccessionRecoveryJobConfiguration.class, RSAccessionRecoveryStepConfiguration.class, RSAccessionRecoveryServiceConfiguration.class, diff --git a/eva-accession-clustering/src/test/resources/backpropagation-test.properties b/eva-accession-clustering/src/test/resources/backpropagation-test.properties index 336a50626..fcdc8ba42 100644 --- a/eva-accession-clustering/src/test/resources/backpropagation-test.properties +++ b/eva-accession-clustering/src/test/resources/backpropagation-test.properties @@ -21,6 +21,10 @@ accessioning.monotonic.rs.blockSize=100000 accessioning.monotonic.rs.blockStartValue=5 accessioning.monotonic.rs.nextBlockInterval=1000000000 +continuous.id.blocks.datasource.driver-class-name=org.hsqldb.jdbc.JDBCDriver +continuous.id.blocks.datasource.jdbc-url=jdbc:hsqldb:mem:db;sql.syntax_pgs=true;DB_CLOSE_DELAY=-1 +continuous.id.blocks.datasource.username=SA +continuous.id.blocks.datasource.password= spring.data.mongodb.database=test-db mongodb.read-preference=primary diff --git a/eva-accession-clustering/src/test/resources/clustering-issuance-test.properties b/eva-accession-clustering/src/test/resources/clustering-issuance-test.properties index d39fc667d..07a0d74bf 100644 --- a/eva-accession-clustering/src/test/resources/clustering-issuance-test.properties +++ b/eva-accession-clustering/src/test/resources/clustering-issuance-test.properties @@ -22,6 +22,11 @@ accessioning.monotonic.rs.blockSize=100000 accessioning.monotonic.rs.blockStartValue=3000000000 accessioning.monotonic.rs.nextBlockInterval=1000000000 +continuous.id.blocks.datasource.driver-class-name=org.hsqldb.jdbc.JDBCDriver +continuous.id.blocks.datasource.jdbc-url=jdbc:hsqldb:mem:db;sql.syntax_pgs=true;DB_CLOSE_DELAY=-1 +continuous.id.blocks.datasource.username=SA +continuous.id.blocks.datasource.password= + spring.data.mongodb.database=test-db mongodb.read-preference=primary diff --git a/eva-accession-clustering/src/test/resources/clustering-pipeline-test.properties b/eva-accession-clustering/src/test/resources/clustering-pipeline-test.properties index cadc78ebb..5b3cd35fd 100644 --- a/eva-accession-clustering/src/test/resources/clustering-pipeline-test.properties +++ b/eva-accession-clustering/src/test/resources/clustering-pipeline-test.properties @@ -22,6 +22,11 @@ accessioning.monotonic.rs.blockSize=100000 accessioning.monotonic.rs.blockStartValue=5 accessioning.monotonic.rs.nextBlockInterval=1000000000 +continuous.id.blocks.datasource.driver-class-name=org.hsqldb.jdbc.JDBCDriver +continuous.id.blocks.datasource.jdbc-url=jdbc:hsqldb:mem:db;sql.syntax_pgs=true;DB_CLOSE_DELAY=-1 +continuous.id.blocks.datasource.username=SA +continuous.id.blocks.datasource.password= + spring.data.mongodb.database=test-db mongodb.read-preference=primary diff --git a/eva-accession-clustering/src/test/resources/clustering-qc-test.properties b/eva-accession-clustering/src/test/resources/clustering-qc-test.properties index 4b654722b..50c6eb203 100644 --- a/eva-accession-clustering/src/test/resources/clustering-qc-test.properties +++ b/eva-accession-clustering/src/test/resources/clustering-qc-test.properties @@ -21,6 +21,11 @@ accessioning.monotonic.rs.blockSize=100000 accessioning.monotonic.rs.blockStartValue=3000000000 accessioning.monotonic.rs.nextBlockInterval=1000000000 +continuous.id.blocks.datasource.driver-class-name=org.hsqldb.jdbc.JDBCDriver +continuous.id.blocks.datasource.jdbc-url=jdbc:hsqldb:mem:db;sql.syntax_pgs=true;DB_CLOSE_DELAY=-1 +continuous.id.blocks.datasource.username=SA +continuous.id.blocks.datasource.password= + spring.data.mongodb.database=test-db mongodb.read-preference=primary diff --git a/eva-accession-clustering/src/test/resources/clustering-writer-test.properties b/eva-accession-clustering/src/test/resources/clustering-writer-test.properties index 652d34dd3..1e68809a3 100644 --- a/eva-accession-clustering/src/test/resources/clustering-writer-test.properties +++ b/eva-accession-clustering/src/test/resources/clustering-writer-test.properties @@ -22,6 +22,11 @@ accessioning.monotonic.rs.blockSize=100000 accessioning.monotonic.rs.blockStartValue=3000000000 accessioning.monotonic.rs.nextBlockInterval=1000000000 +continuous.id.blocks.datasource.driver-class-name=org.hsqldb.jdbc.JDBCDriver +continuous.id.blocks.datasource.jdbc-url=jdbc:hsqldb:mem:db;sql.syntax_pgs=true;DB_CLOSE_DELAY=-1 +continuous.id.blocks.datasource.username=SA +continuous.id.blocks.datasource.password= + spring.data.mongodb.database=test-db mongodb.read-preference=primary diff --git a/eva-accession-clustering/src/test/resources/duplicate-rs-acc-qc-test.properties b/eva-accession-clustering/src/test/resources/duplicate-rs-acc-qc-test.properties index 6c0cf5110..7fcdc30ea 100644 --- a/eva-accession-clustering/src/test/resources/duplicate-rs-acc-qc-test.properties +++ b/eva-accession-clustering/src/test/resources/duplicate-rs-acc-qc-test.properties @@ -23,6 +23,11 @@ accessioning.monotonic.rs.blockSize=100000 accessioning.monotonic.rs.blockStartValue=5 accessioning.monotonic.rs.nextBlockInterval=1000000000 +continuous.id.blocks.datasource.driver-class-name=org.hsqldb.jdbc.JDBCDriver +continuous.id.blocks.datasource.jdbc-url=jdbc:hsqldb:mem:db;sql.syntax_pgs=true;DB_CLOSE_DELAY=-1 +continuous.id.blocks.datasource.username=SA +continuous.id.blocks.datasource.password= + spring.data.mongodb.database=test-db mongodb.read-preference=primary diff --git a/eva-accession-clustering/src/test/resources/merge-split-test.properties b/eva-accession-clustering/src/test/resources/merge-split-test.properties index 3aea699b4..450da55cf 100644 --- a/eva-accession-clustering/src/test/resources/merge-split-test.properties +++ b/eva-accession-clustering/src/test/resources/merge-split-test.properties @@ -20,6 +20,11 @@ accessioning.monotonic.rs.blockSize=100000 accessioning.monotonic.rs.blockStartValue=4 accessioning.monotonic.rs.nextBlockInterval=1000000000 +continuous.id.blocks.datasource.driver-class-name=org.hsqldb.jdbc.JDBCDriver +continuous.id.blocks.datasource.jdbc-url=jdbc:hsqldb:mem:db;sql.syntax_pgs=true;DB_CLOSE_DELAY=-1 +continuous.id.blocks.datasource.username=SA +continuous.id.blocks.datasource.password= + spring.data.mongodb.database=test-db mongodb.read-preference=primary diff --git a/eva-accession-clustering/src/test/resources/rs-accession-recovery.properties b/eva-accession-clustering/src/test/resources/rs-accession-recovery.properties index b5418f3f3..0bd65e138 100644 --- a/eva-accession-clustering/src/test/resources/rs-accession-recovery.properties +++ b/eva-accession-clustering/src/test/resources/rs-accession-recovery.properties @@ -1,13 +1,9 @@ recovery.cutoff.days=14 -spring.datasource.driver-class-name=org.hsqldb.jdbcDriver -spring.datasource.url=jdbc:hsqldb:mem:db;sql.syntax_pgs=true;DB_CLOSE_DELAY=-1 -spring.datasource.username=SA -spring.datasource.password= -spring.sql.init.schema-locations=classpath:test-data/contiguous_id_blocks_schema.sql -spring.sql.init.data-locations=classpath:test-data/rs_accession_recovery_test_data.sql -spring.sql.init.mode=always -spring.jpa.hibernate.ddl-auto=none +continuous.id.blocks.datasource.driver-class-name=org.hsqldb.jdbc.JDBCDriver +continuous.id.blocks.datasource.jdbc-url=jdbc:hsqldb:mem:db;sql.syntax_pgs=true;DB_CLOSE_DELAY=-1 +continuous.id.blocks.datasource.username=SA +continuous.id.blocks.datasource.password= parameters.vcf=src/test/resources/input-files/vcf/aggregated_accessioned.vcf.gz parameters.projectAccession=projectId_1 diff --git a/eva-accession-clustering/src/test/resources/test-data/contiguous_id_blocks_schema.sql b/eva-accession-clustering/src/test/resources/test-data/contiguous_id_blocks_schema.sql index 495f9a4fb..a89a52a6b 100644 --- a/eva-accession-clustering/src/test/resources/test-data/contiguous_id_blocks_schema.sql +++ b/eva-accession-clustering/src/test/resources/test-data/contiguous_id_blocks_schema.sql @@ -1,3 +1,7 @@ +CREATE SEQUENCE IF NOT EXISTS contiguous_id_blocks_seq + START WITH 1 + INCREMENT BY 50; + CREATE TABLE contiguous_id_blocks ( id bigint not NULL, application_instance_id varchar(255) not NULL, diff --git a/eva-accession-core/pom.xml b/eva-accession-core/pom.xml index 9335ac7ea..ffa56fdfe 100644 --- a/eva-accession-core/pom.xml +++ b/eva-accession-core/pom.xml @@ -36,10 +36,6 @@ org.springframework.boot spring-boot-starter-data-mongodb - - org.postgresql - postgresql - uk.ac.ebi.ampt2d accession-commons-mongodb @@ -48,6 +44,11 @@ uk.ac.ebi.ampt2d accession-commons-monotonic-generator-jpa + + org.hsqldb + hsqldb + test + org.springframework.boot spring-boot-testcontainers diff --git a/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/GenericApplication.java b/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/GenericApplication.java index 4053156ff..199dc448a 100644 --- a/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/GenericApplication.java +++ b/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/GenericApplication.java @@ -15,13 +15,22 @@ */ package uk.ac.ebi.eva.accession.core; -import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.autoconfigure.batch.BatchAutoConfiguration; +import org.springframework.boot.autoconfigure.data.mongo.MongoDataAutoConfiguration; +import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration; +import org.springframework.boot.autoconfigure.jdbc.DataSourceTransactionManagerAutoConfiguration; +import org.springframework.boot.autoconfigure.orm.jpa.HibernateJpaAutoConfiguration; import org.springframework.context.ConfigurableApplicationContext; -@EnableBatchProcessing -@SpringBootApplication +@SpringBootApplication(exclude = { + MongoDataAutoConfiguration.class, + DataSourceAutoConfiguration.class, + DataSourceTransactionManagerAutoConfiguration.class, + HibernateJpaAutoConfiguration.class, + BatchAutoConfiguration.class +}) public class GenericApplication { public static void main(String[] args) { diff --git a/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/configuration/ContiguousIdBlocksDataSourceConfiguration.java b/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/configuration/ContiguousIdBlocksDataSourceConfiguration.java new file mode 100644 index 000000000..1cbe609f2 --- /dev/null +++ b/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/configuration/ContiguousIdBlocksDataSourceConfiguration.java @@ -0,0 +1,50 @@ +package uk.ac.ebi.eva.accession.core.configuration; + +import jakarta.persistence.EntityManagerFactory; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.boot.jdbc.DataSourceBuilder; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Primary; +import org.springframework.orm.jpa.JpaTransactionManager; +import org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean; +import org.springframework.orm.jpa.vendor.HibernateJpaVendorAdapter; +import org.springframework.transaction.PlatformTransactionManager; + +import javax.sql.DataSource; +import java.util.Properties; + +@Configuration +public class ContiguousIdBlocksDataSourceConfiguration { + + @Bean + @Primary + @ConfigurationProperties(prefix = "continuous.id.blocks.datasource") + public DataSource continuousIdBlocksDataSource() { + return DataSourceBuilder.create().build(); + } + + @Bean + @Primary + public LocalContainerEntityManagerFactoryBean entityManagerFactory(@Qualifier("continuousIdBlocksDataSource") + DataSource dataSource) { + LocalContainerEntityManagerFactoryBean em = new LocalContainerEntityManagerFactoryBean(); + em.setDataSource(dataSource); + em.setPackagesToScan("uk.ac.ebi.ampt2d.commons.accession.persistence.jpa.monotonic.entities"); + HibernateJpaVendorAdapter vendorAdapter = new HibernateJpaVendorAdapter(); + em.setJpaVendorAdapter(vendorAdapter); + + Properties jpaProperties = new Properties(); + jpaProperties.setProperty("hibernate.physical_naming_strategy", + "org.hibernate.boot.model.naming.CamelCaseToUnderscoresNamingStrategy"); + em.setJpaProperties(jpaProperties); + return em; + } + + @Bean + @Primary + public PlatformTransactionManager transactionManager(EntityManagerFactory entityManagerFactory) { + return new JpaTransactionManager(entityManagerFactory); + } +} \ No newline at end of file diff --git a/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/configuration/InMemoryBatchConfiguration.java b/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/configuration/InMemoryBatchConfiguration.java new file mode 100644 index 000000000..c638cd77b --- /dev/null +++ b/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/configuration/InMemoryBatchConfiguration.java @@ -0,0 +1,50 @@ +package uk.ac.ebi.eva.accession.core.configuration; + +import org.springframework.batch.core.configuration.JobRegistry; +import org.springframework.batch.core.configuration.support.MapJobRegistry; +import org.springframework.batch.core.configuration.support.ScopeConfiguration; +import org.springframework.batch.core.explore.JobExplorer; +import org.springframework.batch.core.launch.JobLauncher; +import org.springframework.batch.core.launch.support.TaskExecutorJobLauncher; +import org.springframework.batch.core.repository.JobRepository; +import org.springframework.batch.core.repository.support.ResourcelessJobRepository; +import org.springframework.batch.support.transaction.ResourcelessTransactionManager; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; +import org.springframework.transaction.PlatformTransactionManager; + +@Configuration +@Import(ScopeConfiguration.class) +public class InMemoryBatchConfiguration { + + public static final String BATCH_TRANSACTION_MANAGER = "batchTransactionManager"; + + @Bean(BATCH_TRANSACTION_MANAGER) + public PlatformTransactionManager batchTransactionManager() { + return new ResourcelessTransactionManager(); + } + + @Bean + public JobRepository jobRepository() { + return new ResourcelessJobRepository(); + } + + @Bean + public JobExplorer jobExplorer() { + return new NoOpJobExplorer(); + } + + @Bean + public JobRegistry jobRegistry() { + return new MapJobRegistry(); + } + + @Bean + public JobLauncher jobLauncher(JobRepository jobRepository) throws Exception { + TaskExecutorJobLauncher launcher = new TaskExecutorJobLauncher(); + launcher.setJobRepository(jobRepository); + launcher.afterPropertiesSet(); + return launcher; + } +} \ No newline at end of file diff --git a/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/configuration/NoOpJobExplorer.java b/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/configuration/NoOpJobExplorer.java new file mode 100644 index 000000000..99e3e4062 --- /dev/null +++ b/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/configuration/NoOpJobExplorer.java @@ -0,0 +1,65 @@ +package uk.ac.ebi.eva.accession.core.configuration; + +import org.springframework.batch.core.JobExecution; +import org.springframework.batch.core.JobInstance; +import org.springframework.batch.core.StepExecution; +import org.springframework.batch.core.explore.JobExplorer; +import org.springframework.lang.Nullable; + +import java.util.List; +import java.util.Set; + +public class NoOpJobExplorer implements JobExplorer { + @Override + public List getJobInstances(String jobName, int start, int count) { + return List.of(); + } + + @Nullable + @Override + public JobExecution getJobExecution(@Nullable Long executionId) { + return null; + } + + @Nullable + @Override + public StepExecution getStepExecution(@Nullable Long jobExecutionId, @Nullable Long stepExecutionId) { + return null; + } + + @Nullable + @Override + public JobInstance getJobInstance(@Nullable Long instanceId) { + return null; + } + + @Override + public List getJobExecutions(JobInstance jobInstance) { + return List.of(); + } + + @Override + public Set findRunningJobExecutions(@Nullable String jobName) { + return Set.of(); + } + + @Override + public List getJobNames() { + return List.of(); + } + + @Override + public List findJobInstancesByJobName(String jobName, int start, int count) { + return List.of(); + } + + @Override + public long getJobInstanceCount(@Nullable String jobName) { + return 0; + } + + @Override + public JobInstance getLastJobInstance(String jobName) { + return null; + } +} \ No newline at end of file diff --git a/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/runner/CommandLineRunnerUtils.java b/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/runner/CommandLineRunnerUtils.java deleted file mode 100644 index 589886bf4..000000000 --- a/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/runner/CommandLineRunnerUtils.java +++ /dev/null @@ -1,96 +0,0 @@ -package uk.ac.ebi.eva.accession.core.runner; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.batch.core.BatchStatus; -import org.springframework.batch.core.Entity; -import org.springframework.batch.core.JobExecution; -import org.springframework.batch.core.JobInstance; -import org.springframework.batch.core.JobParameter; -import org.springframework.batch.core.JobParameters; -import org.springframework.batch.core.JobParametersBuilder; -import org.springframework.batch.core.explore.JobExplorer; -import org.springframework.batch.core.launch.NoSuchJobException; -import org.springframework.batch.core.repository.JobRepository; -import uk.ac.ebi.eva.commons.batch.exception.NoPreviousJobExecutionException; -import uk.ac.ebi.eva.commons.batch.job.JobStatusManager; - -import java.util.Comparator; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -public class CommandLineRunnerUtils { - - private static final Logger logger = LoggerFactory.getLogger(CommandLineRunnerUtils.class); - - private static final String RUN_ID_PARAMETER_NAME = "run.id"; - - public static JobParameters addRunIDToJobParameters(String jobName, JobExplorer jobExplorer, - JobParameters jobParameters) { - JobExecution lastJobExecution = getLastJobExecution(jobName, jobExplorer, jobParameters); - if (lastJobExecution != null) { - Long runIdParameterFromLastExecution = lastJobExecution.getJobParameters() - .getLong(RUN_ID_PARAMETER_NAME); - if (runIdParameterFromLastExecution != 0 && lastJobExecution.getStatus() == BatchStatus.FAILED) { - // Spring Batch 4 uses all job parameters (including run.id) to detect previous instances of a job - see https://github.com/spring-projects/spring-boot/blob/86fb39d5c5f474fe3544159270d4c4e2d01d43ef/spring-boot-project/spring-boot-autoconfigure/src/main/java/org/springframework/boot/autoconfigure/batch/JobLauncherCommandLineRunner.java#L222 - // as opposed to Spring 3 which uses only job name - see https://github.com/spring-projects/spring-boot/blob/541890f0e003a6e346f2234102c97105ab1292ee/spring-boot-autoconfigure/src/main/java/org/springframework/boot/autoconfigure/batch/JobLauncherCommandLineRunner.java#L131 - // Therefore, run.id needs to be supplied for failed jobs in order for the job to be detected and resumed - see https://stackoverflow.com/a/59198742/2601814 - return new JobParametersBuilder(jobParameters) - .addLong(RUN_ID_PARAMETER_NAME, runIdParameterFromLastExecution).toJobParameters(); - } - } - - return jobParameters; - } - - public static JobExecution getLastJobExecution(String jobName, JobExplorer jobExplorer, - JobParameters previousJobParameters) { - long previousJobInstanceCount = getPreviousJobInstanceCount(jobName, jobExplorer); - - List jobInstanceList = jobExplorer.getJobInstances(jobName, 0, (int) previousJobInstanceCount); - List matchingJobExecutions; - - for (JobInstance jobInstance : jobInstanceList) { - matchingJobExecutions = jobExplorer.getJobExecutions(jobInstance); - for (JobExecution jobExecution : matchingJobExecutions) { - if (areParametersEquivalentExceptRunId(jobExecution.getJobParameters(), previousJobParameters)) { - return matchingJobExecutions.stream().max(Comparator.comparingLong(Entity::getId)).get(); - } - } - } - return null; - } - - private static long getPreviousJobInstanceCount(String jobName, JobExplorer jobExplorer) { - try { - return jobExplorer.getJobInstanceCount(jobName); - } catch (NoSuchJobException ex) { - return 0; - } - } - - private static boolean areParametersEquivalentExceptRunId(JobParameters parameters1, JobParameters parameters2) { - Map> firstJobParameterMap = new HashMap<>(parameters1.getParameters()); - Map> secondJobParameterMap = new HashMap<>(parameters2.getParameters()); - if (parameters1.getParameters().containsKey(RUN_ID_PARAMETER_NAME)) { - firstJobParameterMap.remove(RUN_ID_PARAMETER_NAME); - } - if (parameters2.getParameters().containsKey(RUN_ID_PARAMETER_NAME)) { - secondJobParameterMap.remove(RUN_ID_PARAMETER_NAME); - } - return firstJobParameterMap.equals(secondJobParameterMap); - } - - public static void markPreviousJobAsFailed(String jobName, JobRepository jobRepository, JobParameters jobParameters) - throws NoPreviousJobExecutionException { - logger.info("Force restartPreviousExecution of job '" + jobName + "' with parameters: " + jobParameters); - try { - JobStatusManager.markLastJobAsFailed(jobRepository, jobName, jobParameters); - } catch (NoPreviousJobExecutionException ex) { - //Be forgiving if a previous job does not exist: just print a warning message - logger.warn(ex.getMessage()); - } - } - -} diff --git a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/batch/io/ClusteredVariantDeprecationWriterTest.java b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/batch/io/ClusteredVariantDeprecationWriterTest.java index 0e864cedb..77e0c1ce7 100644 --- a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/batch/io/ClusteredVariantDeprecationWriterTest.java +++ b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/batch/io/ClusteredVariantDeprecationWriterTest.java @@ -22,12 +22,12 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.batch.item.Chunk; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.ampt2d.commons.accession.hashing.SHA1HashingFunction; +import uk.ac.ebi.eva.accession.core.configuration.ContiguousIdBlocksDataSourceConfiguration; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.ClusteredVariantAccessioningConfiguration; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.MongoConfiguration; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.SubmittedVariantAccessioningConfiguration; @@ -53,9 +53,9 @@ @ExtendWith(SpringExtension.class) @TestPropertySource("classpath:ss-deprecation-test.properties") -@EnableAutoConfiguration @ContextConfiguration(classes = {MongoConfiguration.class, MongoTestConfiguration.class, - SubmittedVariantAccessioningConfiguration.class, ClusteredVariantAccessioningConfiguration.class}) + SubmittedVariantAccessioningConfiguration.class, ClusteredVariantAccessioningConfiguration.class, + ContiguousIdBlocksDataSourceConfiguration.class}) public class ClusteredVariantDeprecationWriterTest extends MongoTestContainerHelper { private static final String ASSEMBLY = "GCA_000000001.1"; diff --git a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/batch/io/SubmittedVariantDeprecationWriterTest.java b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/batch/io/SubmittedVariantDeprecationWriterTest.java index 8179d4a33..d7c60a26f 100644 --- a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/batch/io/SubmittedVariantDeprecationWriterTest.java +++ b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/batch/io/SubmittedVariantDeprecationWriterTest.java @@ -22,12 +22,12 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.batch.item.Chunk; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.ampt2d.commons.accession.hashing.SHA1HashingFunction; +import uk.ac.ebi.eva.accession.core.configuration.ContiguousIdBlocksDataSourceConfiguration; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.ClusteredVariantAccessioningConfiguration; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.MongoConfiguration; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.SubmittedVariantAccessioningConfiguration; @@ -56,13 +56,11 @@ @ExtendWith(SpringExtension.class) @TestPropertySource("classpath:ss-deprecation-test.properties") -@EnableAutoConfiguration @ContextConfiguration(classes = {MongoConfiguration.class, MongoTestConfiguration.class, - SubmittedVariantAccessioningConfiguration.class, ClusteredVariantAccessioningConfiguration.class}) + SubmittedVariantAccessioningConfiguration.class, ClusteredVariantAccessioningConfiguration.class, + ContiguousIdBlocksDataSourceConfiguration.class}) public class SubmittedVariantDeprecationWriterTest extends MongoTestContainerHelper { - private static final String TEST_DB = "sve-deprecation-test"; - private static final String ASSEMBLY = "GCA_000000001.1"; private static final int TAXONOMY = 60711; diff --git a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/model/eva/ClusteredVariantEntityIntegrationTest.java b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/model/eva/ClusteredVariantEntityIntegrationTest.java index 7d535412e..eafc5dc00 100644 --- a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/model/eva/ClusteredVariantEntityIntegrationTest.java +++ b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/model/eva/ClusteredVariantEntityIntegrationTest.java @@ -20,7 +20,6 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.data.mongodb.core.query.Query; import org.springframework.test.context.ContextConfiguration; @@ -41,7 +40,6 @@ import static org.junit.jupiter.api.Assertions.assertEquals; @ExtendWith(SpringExtension.class) -@EnableAutoConfiguration @ContextConfiguration(classes = {MongoConfiguration.class, MongoTestConfiguration.class}) @TestPropertySource("classpath:test-model.properties") public class ClusteredVariantEntityIntegrationTest extends MongoTestContainerHelper { diff --git a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/model/eva/SubmittedVariantEntityIntegrationTest.java b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/model/eva/SubmittedVariantEntityIntegrationTest.java index 92ed34acd..e8ea13b62 100644 --- a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/model/eva/SubmittedVariantEntityIntegrationTest.java +++ b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/model/eva/SubmittedVariantEntityIntegrationTest.java @@ -20,7 +20,6 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.data.mongodb.core.query.Query; import org.springframework.test.context.ContextConfiguration; @@ -40,7 +39,6 @@ import static org.junit.jupiter.api.Assertions.assertEquals; @ExtendWith(SpringExtension.class) -@EnableAutoConfiguration @ContextConfiguration(classes = {MongoConfiguration.class, MongoTestConfiguration.class}) @TestPropertySource("classpath:test-model.properties") public class SubmittedVariantEntityIntegrationTest extends MongoTestContainerHelper { diff --git a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/repository/nonhuman/eva/SubmittedVariantAccessioningRepositoryTest.java b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/repository/nonhuman/eva/SubmittedVariantAccessioningRepositoryTest.java index bb6d40b05..80137fd19 100644 --- a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/repository/nonhuman/eva/SubmittedVariantAccessioningRepositoryTest.java +++ b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/repository/nonhuman/eva/SubmittedVariantAccessioningRepositoryTest.java @@ -26,10 +26,10 @@ import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.ampt2d.commons.accession.persistence.models.AccessionProjection; import uk.ac.ebi.ampt2d.commons.accession.persistence.mongodb.document.AccessionedDocument; +import uk.ac.ebi.eva.accession.core.configuration.ContiguousIdBlocksDataSourceConfiguration; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.SubmittedVariantAccessioningConfiguration; import uk.ac.ebi.eva.accession.core.model.SubmittedVariant; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; -import uk.ac.ebi.eva.accession.core.test.configuration.JPATestConfiguration; import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; @@ -47,7 +47,7 @@ @ExtendWith(SpringExtension.class) @TestPropertySource("classpath:ss-accession-test.properties") @ContextConfiguration(classes = {SubmittedVariantAccessioningConfiguration.class, MongoTestConfiguration.class, - JPATestConfiguration.class}) + ContiguousIdBlocksDataSourceConfiguration.class}) public class SubmittedVariantAccessioningRepositoryTest extends MongoTestContainerHelper { private static final Long CLUSTERED_VARIANT = null; diff --git a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/service/human/dbsnp/HumanDbsnpClusteredVariantAccessioningServiceTest.java b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/service/human/dbsnp/HumanDbsnpClusteredVariantAccessioningServiceTest.java index 30682e090..ce619d43a 100644 --- a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/service/human/dbsnp/HumanDbsnpClusteredVariantAccessioningServiceTest.java +++ b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/service/human/dbsnp/HumanDbsnpClusteredVariantAccessioningServiceTest.java @@ -28,9 +28,9 @@ import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.ampt2d.commons.accession.core.models.AccessionWrapper; +import uk.ac.ebi.eva.accession.core.configuration.ContiguousIdBlocksDataSourceConfiguration; import uk.ac.ebi.eva.accession.core.model.ClusteredVariant; import uk.ac.ebi.eva.accession.core.model.IClusteredVariant; -import uk.ac.ebi.eva.accession.core.test.configuration.JPATestConfiguration; import uk.ac.ebi.eva.accession.core.test.configuration.human.MongoHumanTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.core.utils.MongoTestDataLoader; @@ -45,7 +45,7 @@ @ExtendWith(SpringExtension.class) @TestPropertySource("classpath:ss-accession-test.properties") -@ContextConfiguration(classes = {MongoHumanTestConfiguration.class, JPATestConfiguration.class}) +@ContextConfiguration(classes = {MongoHumanTestConfiguration.class, ContiguousIdBlocksDataSourceConfiguration.class}) public class HumanDbsnpClusteredVariantAccessioningServiceTest extends MongoTestContainerHelper { private static final Long HUMAN_ACTIVE_RS_ID_1 = 1118L; diff --git a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/service/nonhuman/SubmittedVariantAccessioningServiceTest.java b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/service/nonhuman/SubmittedVariantAccessioningServiceTest.java index 9a1b6c91a..6229deb42 100644 --- a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/service/nonhuman/SubmittedVariantAccessioningServiceTest.java +++ b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/service/nonhuman/SubmittedVariantAccessioningServiceTest.java @@ -22,9 +22,12 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.io.ClassPathResource; import org.springframework.core.io.ResourceLoader; import org.springframework.data.mongodb.MongoDatabaseFactory; import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.jdbc.datasource.init.ResourceDatabasePopulator; +import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit.jupiter.SpringExtension; @@ -39,6 +42,7 @@ import uk.ac.ebi.ampt2d.commons.accession.core.models.GetOrCreateAccessionWrapper; import uk.ac.ebi.ampt2d.commons.accession.core.models.IEvent; import uk.ac.ebi.ampt2d.commons.accession.hashing.SHA1HashingFunction; +import uk.ac.ebi.eva.accession.core.configuration.ContiguousIdBlocksDataSourceConfiguration; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.SubmittedVariantAccessioningConfiguration; import uk.ac.ebi.eva.accession.core.generators.DbsnpMonotonicAccessionGenerator; import uk.ac.ebi.eva.accession.core.model.ISubmittedVariant; @@ -48,12 +52,15 @@ import uk.ac.ebi.eva.accession.core.service.nonhuman.dbsnp.DbsnpSubmittedVariantMonotonicAccessioningService; import uk.ac.ebi.eva.accession.core.service.nonhuman.eva.SubmittedVariantInactiveService; import uk.ac.ebi.eva.accession.core.summary.SubmittedVariantSummaryFunction; -import uk.ac.ebi.eva.accession.core.test.configuration.JPATestConfiguration; import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.core.utils.MongoTestDataLoader; import uk.ac.ebi.eva.commons.core.models.contigalias.ContigNamingConvention; +import javax.sql.DataSource; +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; import java.time.LocalDateTime; import java.time.Month; import java.util.ArrayList; @@ -76,7 +83,8 @@ @ExtendWith(SpringExtension.class) @TestPropertySource("classpath:ss-accession-test.properties") @ContextConfiguration(classes = {SubmittedVariantAccessioningConfiguration.class, MongoTestConfiguration.class, - JPATestConfiguration.class}) + ContiguousIdBlocksDataSourceConfiguration.class}) +@DirtiesContext(classMode = DirtiesContext.ClassMode.BEFORE_EACH_TEST_METHOD) public class SubmittedVariantAccessioningServiceTest extends MongoTestContainerHelper { private static String TEST_APPLICATION_INSTANCE_ID = "test-application-instance-id"; @@ -130,6 +138,9 @@ public class SubmittedVariantAccessioningServiceTest extends MongoTestContainerH @Autowired private Long accessioningMonotonicInitSs; + @Autowired + private DataSource dataSource; + @Autowired private MongoDatabaseFactory mongoDbFactory; @@ -140,7 +151,7 @@ public class SubmittedVariantAccessioningServiceTest extends MongoTestContainerH private MongoTemplate mongoTemplate; @BeforeEach - public void setUp() { + public void setUp() throws SQLException { mongoTemplate.getDb().drop(); submittedVariant = new SubmittedVariant("GCA_000003055.3", 9913, PROJECT, "21", 20800319, "C", "T", CLUSTERED_VARIANT, DEFAULT_SUPPORTED_BY_EVIDENCE, @@ -160,6 +171,14 @@ public void setUp() { submittedVariantModified = new SubmittedVariant("GCA_000003055.3", 9913, PROJECT, "21", 20800319, "C", "TCTC", CLUSTERED_VARIANT, DEFAULT_SUPPORTED_BY_EVIDENCE, DEFAULT_ASSEMBLY_MATCH, true, DEFAULT_VALIDATED, null); + + try (Connection conn = dataSource.getConnection(); + Statement stmt = conn.createStatement()) { + stmt.execute("DROP TABLE IF EXISTS contiguous_id_blocks"); + } + ResourceDatabasePopulator populator = new ResourceDatabasePopulator(); + populator.addScript(new ClassPathResource("test-data/contiguous_id_blocks_schema.sql")); + populator.execute(dataSource); } @AfterEach @@ -169,6 +188,7 @@ public void tearDown() { @Test public void sameAccessionsAreReturnedForIdenticalVariants() throws AccessionCouldNotBeGeneratedException { + List variants = Arrays.asList( new SubmittedVariant("assembly", 1111, "project", "contig_1", 100, "ref", "alt", CLUSTERED_VARIANT, DEFAULT_SUPPORTED_BY_EVIDENCE, DEFAULT_ASSEMBLY_MATCH, ALLELES_MATCH, diff --git a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/test/configuration/JPATestConfiguration.java b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/test/configuration/JPATestConfiguration.java deleted file mode 100644 index 66a846d06..000000000 --- a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/test/configuration/JPATestConfiguration.java +++ /dev/null @@ -1,41 +0,0 @@ -package uk.ac.ebi.eva.accession.core.test.configuration; - -import jakarta.persistence.EntityManagerFactory; -import org.springframework.context.annotation.Bean; -import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseBuilder; -import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseType; -import org.springframework.orm.jpa.JpaTransactionManager; -import org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean; -import org.springframework.orm.jpa.vendor.HibernateJpaVendorAdapter; -import org.springframework.transaction.PlatformTransactionManager; - -import javax.sql.DataSource; - -public class JPATestConfiguration { - @Bean - public DataSource dataSource() { - return new EmbeddedDatabaseBuilder() - .setType(EmbeddedDatabaseType.HSQL) - .build(); - } - - @Bean - public LocalContainerEntityManagerFactoryBean entityManagerFactory(DataSource dataSource) { - LocalContainerEntityManagerFactoryBean em = new LocalContainerEntityManagerFactoryBean(); - em.setDataSource(dataSource); - // Package where ContiguousIdBlockEntity (the JPA entity) lives - em.setPackagesToScan( - "uk.ac.ebi.ampt2d.commons.accession.persistence.jpa.monotonic.entities" - ); - HibernateJpaVendorAdapter vendorAdapter = new HibernateJpaVendorAdapter(); - vendorAdapter.setGenerateDdl(true); // auto-creates the schema in H2 - vendorAdapter.setShowSql(false); - em.setJpaVendorAdapter(vendorAdapter); - return em; - } - - @Bean - public PlatformTransactionManager transactionManager(EntityManagerFactory entityManagerFactory) { - return new JpaTransactionManager(entityManagerFactory); - } -} diff --git a/eva-accession-core/src/test/resources/properties/ss-accession-test.properties b/eva-accession-core/src/test/resources/properties/ss-accession-test.properties index b4151457a..5c9e591cc 100644 --- a/eva-accession-core/src/test/resources/properties/ss-accession-test.properties +++ b/eva-accession-core/src/test/resources/properties/ss-accession-test.properties @@ -8,6 +8,11 @@ accessioning.monotonic.test-rs.blockSize=100000 accessioning.monotonic.test-rs.blockStartValue=3000000000 accessioning.monotonic.test-rs.nextBlockInterval=1000000000 +continuous.id.blocks.datasource.driver-class-name=org.hsqldb.jdbc.JDBCDriver +continuous.id.blocks.datasource.jdbc-url=jdbc:hsqldb:mem:db;sql.syntax_pgs=true;DB_CLOSE_DELAY=-1 +continuous.id.blocks.datasource.username=SA +continuous.id.blocks.datasource.password= + spring.data.mongodb.database=submitted-variants-test # This symbolic variable is required because # this will be replaced with a default value "localhost" when run locally diff --git a/eva-accession-core/src/test/resources/properties/ss-deprecation-test.properties b/eva-accession-core/src/test/resources/properties/ss-deprecation-test.properties index 50d8e3cd5..f9a467bb5 100644 --- a/eva-accession-core/src/test/resources/properties/ss-deprecation-test.properties +++ b/eva-accession-core/src/test/resources/properties/ss-deprecation-test.properties @@ -9,6 +9,11 @@ accessioning.monotonic.test-rs.blockSize=100000 accessioning.monotonic.test-rs.blockStartValue=4 accessioning.monotonic.test-rs.nextBlockInterval=1000000000 +continuous.id.blocks.datasource.driver-class-name=org.hsqldb.jdbc.JDBCDriver +continuous.id.blocks.datasource.jdbc-url=jdbc:hsqldb:mem:db;sql.syntax_pgs=true;DB_CLOSE_DELAY=-1 +continuous.id.blocks.datasource.username=SA +continuous.id.blocks.datasource.password= + spring.data.mongodb.database=sve-deprecation-test # This symbolic variable is required because # this will be replaced with a default value "localhost" when run locally diff --git a/eva-accession-core/src/test/resources/test-data/contiguous_id_blocks_schema.sql b/eva-accession-core/src/test/resources/test-data/contiguous_id_blocks_schema.sql new file mode 100644 index 000000000..32dfe369d --- /dev/null +++ b/eva-accession-core/src/test/resources/test-data/contiguous_id_blocks_schema.sql @@ -0,0 +1,14 @@ +CREATE SEQUENCE IF NOT EXISTS contiguous_id_blocks_seq + START WITH 1 + INCREMENT BY 50; + +CREATE TABLE IF NOT EXISTS contiguous_id_blocks ( + id bigint not NULL, + application_instance_id varchar(255) not NULL, + category_id varchar(255) not NULL, + first_value bigint not NULL, + last_committed bigint not NULL, + last_value bigint not NULL, + reserved boolean NOT NULL, + last_updated_timestamp timestamp NOT NULL +); diff --git a/eva-accession-deprecate/pom.xml b/eva-accession-deprecate/pom.xml index 71cf08411..da20d7e0a 100644 --- a/eva-accession-deprecate/pom.xml +++ b/eva-accession-deprecate/pom.xml @@ -28,6 +28,11 @@ xstream + + org.postgresql + postgresql + + uk.ac.ebi.eva eva-accession-core @@ -36,6 +41,12 @@ test + + org.hsqldb + hsqldb + test + + org.springframework.boot spring-boot-testcontainers diff --git a/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/Application.java b/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/Application.java index 1a67c884d..498c87682 100644 --- a/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/Application.java +++ b/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/Application.java @@ -15,13 +15,26 @@ */ package uk.ac.ebi.eva.accession.deprecate; -import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.autoconfigure.batch.BatchAutoConfiguration; +import org.springframework.boot.autoconfigure.data.mongo.MongoDataAutoConfiguration; +import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration; +import org.springframework.boot.autoconfigure.jdbc.DataSourceTransactionManagerAutoConfiguration; +import org.springframework.boot.autoconfigure.orm.jpa.HibernateJpaAutoConfiguration; import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.context.annotation.Import; +import uk.ac.ebi.eva.accession.core.configuration.ContiguousIdBlocksDataSourceConfiguration; +import uk.ac.ebi.eva.accession.core.configuration.InMemoryBatchConfiguration; -@EnableBatchProcessing -@SpringBootApplication +@SpringBootApplication(exclude = { + MongoDataAutoConfiguration.class, + DataSourceAutoConfiguration.class, + DataSourceTransactionManagerAutoConfiguration.class, + HibernateJpaAutoConfiguration.class, + BatchAutoConfiguration.class +}) +@Import({InMemoryBatchConfiguration.class, ContiguousIdBlocksDataSourceConfiguration.class}) public class Application { public static void main(String[] args) { diff --git a/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/batch/jobs/DeprecateStudySubmittedVariantsJobConfiguration.java b/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/batch/jobs/DeprecateStudySubmittedVariantsJobConfiguration.java index ebe7fa92b..1e23f8e17 100644 --- a/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/batch/jobs/DeprecateStudySubmittedVariantsJobConfiguration.java +++ b/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/batch/jobs/DeprecateStudySubmittedVariantsJobConfiguration.java @@ -17,9 +17,7 @@ import org.springframework.batch.core.Job; import org.springframework.batch.core.Step; -import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.batch.core.job.builder.JobBuilder; -import org.springframework.batch.core.launch.support.RunIdIncrementer; import org.springframework.batch.core.repository.JobRepository; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; @@ -28,7 +26,6 @@ import uk.ac.ebi.eva.accession.deprecate.configuration.BeanNames; @Configuration -@EnableBatchProcessing public class DeprecateStudySubmittedVariantsJobConfiguration { @Autowired @@ -38,7 +35,6 @@ public class DeprecateStudySubmittedVariantsJobConfiguration { @Bean(BeanNames.DEPRECATE_STUDY_SUBMITTED_VARIANTS_JOB) public Job accessionReleaseJob(JobRepository jobRepository) { return new JobBuilder(BeanNames.DEPRECATE_STUDY_SUBMITTED_VARIANTS_JOB, jobRepository) - .incrementer(new RunIdIncrementer()) .start(deprecateStudySubmittedVariantsStep) .build(); } diff --git a/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/batch/jobs/DeprecateSubmittedVariantsFromFileJobConfiguration.java b/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/batch/jobs/DeprecateSubmittedVariantsFromFileJobConfiguration.java index 05958a588..14b09a85c 100644 --- a/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/batch/jobs/DeprecateSubmittedVariantsFromFileJobConfiguration.java +++ b/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/batch/jobs/DeprecateSubmittedVariantsFromFileJobConfiguration.java @@ -17,9 +17,7 @@ import org.springframework.batch.core.Job; import org.springframework.batch.core.Step; -import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.batch.core.job.builder.JobBuilder; -import org.springframework.batch.core.launch.support.RunIdIncrementer; import org.springframework.batch.core.repository.JobRepository; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; @@ -28,7 +26,6 @@ import uk.ac.ebi.eva.accession.deprecate.configuration.BeanNames; @Configuration -@EnableBatchProcessing public class DeprecateSubmittedVariantsFromFileJobConfiguration { @Autowired @@ -38,7 +35,6 @@ public class DeprecateSubmittedVariantsFromFileJobConfiguration { @Bean(BeanNames.DEPRECATE_SUBMITTED_VARIANTS_FROM_FILE_JOB) public Job deprecateStudySubmittedVariantsFromFileJob(JobRepository jobRepository) { return new JobBuilder(BeanNames.DEPRECATE_SUBMITTED_VARIANTS_FROM_FILE_JOB, jobRepository) - .incrementer(new RunIdIncrementer()) .start(deprecateSubmittedVariantsFromFileStep) .build(); } diff --git a/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/batch/steps/DeprecateStudySubmittedVariantsStepConfiguration.java b/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/batch/steps/DeprecateStudySubmittedVariantsStepConfiguration.java index 37261c120..3fa8ec836 100644 --- a/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/batch/steps/DeprecateStudySubmittedVariantsStepConfiguration.java +++ b/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/batch/steps/DeprecateStudySubmittedVariantsStepConfiguration.java @@ -17,7 +17,6 @@ import org.springframework.batch.core.Step; import org.springframework.batch.core.StepExecutionListener; -import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.batch.core.repository.JobRepository; import org.springframework.batch.core.step.builder.StepBuilder; import org.springframework.batch.core.step.tasklet.TaskletStep; @@ -32,8 +31,9 @@ import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; import uk.ac.ebi.eva.accession.deprecate.configuration.BeanNames; +import static uk.ac.ebi.eva.accession.core.configuration.InMemoryBatchConfiguration.BATCH_TRANSACTION_MANAGER; + @Configuration -@EnableBatchProcessing public class DeprecateStudySubmittedVariantsStepConfiguration { @Autowired @@ -50,6 +50,7 @@ public class DeprecateStudySubmittedVariantsStepConfiguration { @Bean(BeanNames.DEPRECATE_STUDY_SUBMITTED_VARIANTS_STEP) public Step deprecateClusteredVariantsStep(JobRepository jobRepository, + @Qualifier(BATCH_TRANSACTION_MANAGER) PlatformTransactionManager transactionManager, SimpleCompletionPolicy chunkSizeCompletionPolicy) { TaskletStep step = new StepBuilder(BeanNames.DEPRECATE_STUDY_SUBMITTED_VARIANTS_STEP, jobRepository) diff --git a/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/batch/steps/DeprecateSubmittedVariantsFromFileStepConfiguration.java b/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/batch/steps/DeprecateSubmittedVariantsFromFileStepConfiguration.java index aad2dfedd..c07280dac 100644 --- a/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/batch/steps/DeprecateSubmittedVariantsFromFileStepConfiguration.java +++ b/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/batch/steps/DeprecateSubmittedVariantsFromFileStepConfiguration.java @@ -17,7 +17,6 @@ import org.springframework.batch.core.Step; import org.springframework.batch.core.StepExecutionListener; -import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.batch.core.repository.JobRepository; import org.springframework.batch.core.step.builder.StepBuilder; import org.springframework.batch.core.step.tasklet.TaskletStep; @@ -32,8 +31,9 @@ import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; import uk.ac.ebi.eva.accession.deprecate.configuration.BeanNames; +import static uk.ac.ebi.eva.accession.core.configuration.InMemoryBatchConfiguration.BATCH_TRANSACTION_MANAGER; + @Configuration -@EnableBatchProcessing public class DeprecateSubmittedVariantsFromFileStepConfiguration { @Autowired @@ -50,6 +50,7 @@ public class DeprecateSubmittedVariantsFromFileStepConfiguration { @Bean(BeanNames.DEPRECATE_SUBMITTED_VARIANTS_FROM_FILE_STEP) public Step deprecateSubmittedVariantsFromFileStep(JobRepository jobRepository, + @Qualifier(BATCH_TRANSACTION_MANAGER) PlatformTransactionManager transactionManager, SimpleCompletionPolicy chunkSizeCompletionPolicy) { TaskletStep step = new StepBuilder(BeanNames.DEPRECATE_SUBMITTED_VARIANTS_FROM_FILE_STEP, jobRepository) diff --git a/eva-accession-deprecate/src/main/resources/application.properties b/eva-accession-deprecate/src/main/resources/application.properties index 798e22105..15712493e 100644 --- a/eva-accession-deprecate/src/main/resources/application.properties +++ b/eva-accession-deprecate/src/main/resources/application.properties @@ -22,12 +22,6 @@ spring.data.mongodb.password= spring.data.mongodb.authentication-database=admin mongodb.read-preference=primary -spring.datasource.driver-class-name=org.postgresql.Driver -spring.datasource.url= -spring.datasource.username= -spring.datasource.password= -spring.datasource.tomcat.max-active=3 - #See https://github.com/spring-projects/spring-boot/wiki/Spring-Boot-2.1-Release-Notes#bean-overriding spring.main.allow-bean-definition-overriding=true #As this is a spring batch application, disable the embedded tomcat. This is the new way to do that for spring 2. diff --git a/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/batch/io/StudySubmittedVariantsReaderTest.java b/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/batch/io/StudySubmittedVariantsReaderTest.java index ea6b98198..e552b02b1 100644 --- a/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/batch/io/StudySubmittedVariantsReaderTest.java +++ b/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/batch/io/StudySubmittedVariantsReaderTest.java @@ -27,8 +27,8 @@ import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.MongoConfiguration; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; -import uk.ac.ebi.eva.accession.deprecate.test.configuration.MongoTestConfiguration; import java.util.ArrayList; import java.util.List; diff --git a/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/configuration/jobs/DeprecateStudySubmittedVariantsJobConfigurationTest.java b/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/configuration/jobs/DeprecateStudySubmittedVariantsJobConfigurationTest.java index 2c44eb795..d2b3d6cd7 100644 --- a/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/configuration/jobs/DeprecateStudySubmittedVariantsJobConfigurationTest.java +++ b/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/configuration/jobs/DeprecateStudySubmittedVariantsJobConfigurationTest.java @@ -29,16 +29,14 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.data.mongodb.core.MongoTemplate; -import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit.jupiter.SpringExtension; -import uk.ac.ebi.eva.accession.core.runner.CommandLineRunnerUtils; +import uk.ac.ebi.eva.accession.core.configuration.ContiguousIdBlocksDataSourceConfiguration; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.deprecate.MongoTestDatabaseSetup; -import uk.ac.ebi.eva.accession.deprecate.test.configuration.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.accession.deprecate.test.configuration.BatchTestConfiguration; -import uk.ac.ebi.eva.accession.deprecate.test.configuration.MongoTestConfiguration; import java.util.Collection; import java.util.Collections; @@ -46,14 +44,12 @@ import java.util.stream.Collectors; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotEquals; -import static uk.ac.ebi.eva.accession.deprecate.configuration.BeanNames.DEPRECATE_STUDY_SUBMITTED_VARIANTS_JOB; import static uk.ac.ebi.eva.accession.deprecate.configuration.BeanNames.DEPRECATE_STUDY_SUBMITTED_VARIANTS_STEP; import static uk.ac.ebi.eva.accession.deprecate.test.configuration.BatchTestConfiguration.JOB_LAUNCHER_FROM_MONGO; @ExtendWith(SpringExtension.class) @ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class, - BatchJobRepositoryTestConfiguration.class}) + ContiguousIdBlocksDataSourceConfiguration.class}) @TestPropertySource("classpath:study-submitted-variants-test.properties") public class DeprecateStudySubmittedVariantsJobConfigurationTest extends MongoTestContainerHelper { @Autowired @@ -101,31 +97,4 @@ private void assertStepsExecuted(List expectedSteps, JobExecution jobExecution) List steps = stepExecutions.stream().map(StepExecution::getStepName).collect(Collectors.toList()); assertEquals(expectedSteps, steps); } - - @Test - @DirtiesContext - public void restartCompletedJobThatIsAlreadyInTheRepository() throws Exception { - JobExecution jobExecution = jobLauncherTestUtilsFromMongo.launchJob(); - - List expectedSteps = Collections.singletonList(DEPRECATE_STUDY_SUBMITTED_VARIANTS_STEP); - assertStepsExecuted(expectedSteps, jobExecution); - assertEquals(BatchStatus.COMPLETED, jobExecution.getStatus()); - MongoTestDatabaseSetup.assertPostDeprecationDatabaseState(this.mongoTemplate); - - long instanceIdFirstJob = CommandLineRunnerUtils.getLastJobExecution(DEPRECATE_STUDY_SUBMITTED_VARIANTS_JOB, - jobExplorer, - jobExecution.getJobParameters()) - .getJobInstance().getInstanceId(); - - jobExecution = jobLauncherTestUtilsFromMongo.launchJob(); - expectedSteps = Collections.singletonList(DEPRECATE_STUDY_SUBMITTED_VARIANTS_STEP); - assertStepsExecuted(expectedSteps, jobExecution); - assertEquals(BatchStatus.COMPLETED, jobExecution.getStatus()); - long instanceIdSecondJob = CommandLineRunnerUtils.getLastJobExecution(DEPRECATE_STUDY_SUBMITTED_VARIANTS_JOB, - jobExplorer, - jobExecution.getJobParameters()) - .getJobInstance().getInstanceId(); - assertNotEquals(instanceIdSecondJob, instanceIdFirstJob); - MongoTestDatabaseSetup.assertPostDeprecationDatabaseState(this.mongoTemplate); - } } diff --git a/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/configuration/jobs/DeprecateSubmittedVariantsFromFileJobConfigurationTest.java b/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/configuration/jobs/DeprecateSubmittedVariantsFromFileJobConfigurationTest.java index 693ff64f1..819df4c0b 100644 --- a/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/configuration/jobs/DeprecateSubmittedVariantsFromFileJobConfigurationTest.java +++ b/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/configuration/jobs/DeprecateSubmittedVariantsFromFileJobConfigurationTest.java @@ -29,16 +29,14 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.data.mongodb.core.MongoTemplate; -import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit.jupiter.SpringExtension; -import uk.ac.ebi.eva.accession.core.runner.CommandLineRunnerUtils; +import uk.ac.ebi.eva.accession.core.configuration.ContiguousIdBlocksDataSourceConfiguration; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.deprecate.MongoTestDatabaseSetup; -import uk.ac.ebi.eva.accession.deprecate.test.configuration.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.accession.deprecate.test.configuration.BatchTestConfiguration; -import uk.ac.ebi.eva.accession.deprecate.test.configuration.MongoTestConfiguration; import java.util.Collection; import java.util.Collections; @@ -46,14 +44,12 @@ import java.util.stream.Collectors; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotEquals; -import static uk.ac.ebi.eva.accession.deprecate.configuration.BeanNames.DEPRECATE_SUBMITTED_VARIANTS_FROM_FILE_JOB; import static uk.ac.ebi.eva.accession.deprecate.configuration.BeanNames.DEPRECATE_SUBMITTED_VARIANTS_FROM_FILE_STEP; import static uk.ac.ebi.eva.accession.deprecate.test.configuration.BatchTestConfiguration.JOB_LAUNCHER_FROM_FILE; @ExtendWith(SpringExtension.class) @ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class, - BatchJobRepositoryTestConfiguration.class}) + ContiguousIdBlocksDataSourceConfiguration.class}) @TestPropertySource("classpath:deprecate-submitted-variants-from-file-test.properties") public class DeprecateSubmittedVariantsFromFileJobConfigurationTest extends MongoTestContainerHelper { @Autowired @@ -101,31 +97,4 @@ private void assertStepsExecuted(List expectedSteps, JobExecution jobExecution) List steps = stepExecutions.stream().map(StepExecution::getStepName).collect(Collectors.toList()); assertEquals(expectedSteps, steps); } - - @Test - @DirtiesContext - public void restartCompletedJobThatIsAlreadyInTheRepository() throws Exception { - JobExecution jobExecution = jobLauncherTestUtilsFromFile.launchJob(); - - List expectedSteps = Collections.singletonList(DEPRECATE_SUBMITTED_VARIANTS_FROM_FILE_STEP); - assertStepsExecuted(expectedSteps, jobExecution); - assertEquals(BatchStatus.COMPLETED, jobExecution.getStatus()); - MongoTestDatabaseSetup.assertPostDeprecationDatabaseState(this.mongoTemplate); - - long instanceIdFirstJob = CommandLineRunnerUtils.getLastJobExecution(DEPRECATE_SUBMITTED_VARIANTS_FROM_FILE_JOB, - jobExplorer, - jobExecution.getJobParameters()) - .getJobInstance().getInstanceId(); - - jobExecution = jobLauncherTestUtilsFromFile.launchJob(); - expectedSteps = Collections.singletonList(DEPRECATE_SUBMITTED_VARIANTS_FROM_FILE_STEP); - assertStepsExecuted(expectedSteps, jobExecution); - assertEquals(BatchStatus.COMPLETED, jobExecution.getStatus()); - long instanceIdSecondJob = CommandLineRunnerUtils.getLastJobExecution(DEPRECATE_SUBMITTED_VARIANTS_FROM_FILE_JOB, - jobExplorer, - jobExecution.getJobParameters()) - .getJobInstance().getInstanceId(); - assertNotEquals(instanceIdSecondJob, instanceIdFirstJob); - MongoTestDatabaseSetup.assertPostDeprecationDatabaseState(this.mongoTemplate); - } } diff --git a/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/configuration/steps/DeprecateStudySubmittedVariantsStepConfigurationTest.java b/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/configuration/steps/DeprecateStudySubmittedVariantsStepConfigurationTest.java index 67ad9f95a..7328fc11a 100644 --- a/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/configuration/steps/DeprecateStudySubmittedVariantsStepConfigurationTest.java +++ b/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/configuration/steps/DeprecateStudySubmittedVariantsStepConfigurationTest.java @@ -28,19 +28,19 @@ import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit.jupiter.SpringExtension; +import uk.ac.ebi.eva.accession.core.configuration.ContiguousIdBlocksDataSourceConfiguration; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.deprecate.MongoTestDatabaseSetup; import uk.ac.ebi.eva.accession.deprecate.configuration.BeanNames; -import uk.ac.ebi.eva.accession.deprecate.test.configuration.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.accession.deprecate.test.configuration.BatchTestConfiguration; -import uk.ac.ebi.eva.accession.deprecate.test.configuration.MongoTestConfiguration; import static org.junit.jupiter.api.Assertions.assertEquals; import static uk.ac.ebi.eva.accession.deprecate.test.configuration.BatchTestConfiguration.JOB_LAUNCHER_FROM_MONGO; @ExtendWith(SpringExtension.class) @ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class, - BatchJobRepositoryTestConfiguration.class}) + ContiguousIdBlocksDataSourceConfiguration.class}) @TestPropertySource("classpath:study-submitted-variants-test.properties") public class DeprecateStudySubmittedVariantsStepConfigurationTest extends MongoTestContainerHelper { @Autowired diff --git a/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/configuration/steps/DeprecateSubmittedVariantsFromFileStepConfigurationTest.java b/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/configuration/steps/DeprecateSubmittedVariantsFromFileStepConfigurationTest.java index 05ab6f356..6bcae4b68 100644 --- a/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/configuration/steps/DeprecateSubmittedVariantsFromFileStepConfigurationTest.java +++ b/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/configuration/steps/DeprecateSubmittedVariantsFromFileStepConfigurationTest.java @@ -28,19 +28,19 @@ import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit.jupiter.SpringExtension; +import uk.ac.ebi.eva.accession.core.configuration.ContiguousIdBlocksDataSourceConfiguration; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.deprecate.MongoTestDatabaseSetup; import uk.ac.ebi.eva.accession.deprecate.configuration.BeanNames; -import uk.ac.ebi.eva.accession.deprecate.test.configuration.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.accession.deprecate.test.configuration.BatchTestConfiguration; -import uk.ac.ebi.eva.accession.deprecate.test.configuration.MongoTestConfiguration; import static org.junit.jupiter.api.Assertions.assertEquals; import static uk.ac.ebi.eva.accession.deprecate.test.configuration.BatchTestConfiguration.JOB_LAUNCHER_FROM_FILE; @ExtendWith(SpringExtension.class) @ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class, - BatchJobRepositoryTestConfiguration.class}) + ContiguousIdBlocksDataSourceConfiguration.class}) @TestPropertySource("classpath:deprecate-submitted-variants-from-file-test.properties") public class DeprecateSubmittedVariantsFromFileStepConfigurationTest extends MongoTestContainerHelper { @Autowired diff --git a/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/test/configuration/BatchJobRepositoryTestConfiguration.java b/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/test/configuration/BatchJobRepositoryTestConfiguration.java deleted file mode 100644 index 854541a72..000000000 --- a/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/test/configuration/BatchJobRepositoryTestConfiguration.java +++ /dev/null @@ -1,22 +0,0 @@ -package uk.ac.ebi.eva.accession.deprecate.test.configuration; - -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Primary; -import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseBuilder; -import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseType; - -import javax.sql.DataSource; - -@Configuration -public class BatchJobRepositoryTestConfiguration { - - @Bean - @Primary - public DataSource dataSource() { - return new EmbeddedDatabaseBuilder() - .setType(EmbeddedDatabaseType.HSQL) - .addScript("org/springframework/batch/core/schema-hsqldb.sql") - .build(); - } -} \ No newline at end of file diff --git a/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/test/configuration/BatchTestConfiguration.java b/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/test/configuration/BatchTestConfiguration.java index f30f6de12..bc56c123d 100644 --- a/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/test/configuration/BatchTestConfiguration.java +++ b/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/test/configuration/BatchTestConfiguration.java @@ -20,9 +20,10 @@ import org.springframework.batch.core.repository.JobRepository; import org.springframework.batch.test.JobLauncherTestUtils; import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Import; +import org.springframework.web.client.RestTemplate; +import uk.ac.ebi.eva.accession.core.configuration.InMemoryBatchConfiguration; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.MongoConfiguration; import uk.ac.ebi.eva.accession.deprecate.configuration.InputParametersConfiguration; import uk.ac.ebi.eva.accession.deprecate.configuration.batch.io.StudySubmittedVariantsFileReaderConfiguration; @@ -38,8 +39,8 @@ import static uk.ac.ebi.eva.accession.deprecate.configuration.BeanNames.DEPRECATE_STUDY_SUBMITTED_VARIANTS_JOB; import static uk.ac.ebi.eva.accession.deprecate.configuration.BeanNames.DEPRECATE_SUBMITTED_VARIANTS_FROM_FILE_JOB; -@EnableAutoConfiguration -@Import({MongoConfiguration.class, +@Import({InMemoryBatchConfiguration.class, + MongoConfiguration.class, InputParametersConfiguration.class, ChunkSizeCompletionPolicyConfiguration.class, StudySubmittedVariantsReaderConfiguration.class, @@ -75,4 +76,11 @@ public JobLauncherTestUtils jobLauncherTestUtilsFromFile(JobLauncher jobLauncher return utils; } + + @Bean(name = "COUNT_STATS_REST_TEMPLATE") + public RestTemplate countStatsRestTemplate() { + return new RestTemplate(); + } + + } diff --git a/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/test/configuration/MongoTestConfiguration.java b/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/test/configuration/MongoTestConfiguration.java deleted file mode 100644 index 08c2a4774..000000000 --- a/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/test/configuration/MongoTestConfiguration.java +++ /dev/null @@ -1,105 +0,0 @@ -/* - * - * Copyright 2022 EMBL - European Bioinformatics Institute - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -package uk.ac.ebi.eva.accession.deprecate.test.configuration; - -import com.mongodb.ReadPreference; -import com.mongodb.WriteConcern; -import com.mongodb.client.MongoClient; -import com.mongodb.client.MongoClients; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.domain.EntityScan; -import org.springframework.boot.test.autoconfigure.data.mongo.AutoConfigureDataMongo; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Primary; -import org.springframework.data.mongodb.MongoDatabaseFactory; -import org.springframework.data.mongodb.config.EnableMongoAuditing; -import org.springframework.data.mongodb.core.MongoTemplate; -import org.springframework.data.mongodb.core.SimpleMongoClientDatabaseFactory; -import org.springframework.data.mongodb.core.WriteResultChecking; -import org.springframework.data.mongodb.core.convert.DefaultDbRefResolver; -import org.springframework.data.mongodb.core.convert.DefaultMongoTypeMapper; -import org.springframework.data.mongodb.core.convert.MappingMongoConverter; -import org.springframework.data.mongodb.core.convert.MongoCustomConversions; -import org.springframework.data.mongodb.core.mapping.MongoMappingContext; -import org.springframework.data.mongodb.repository.config.EnableMongoRepositories; - -import java.util.Collections; - -@Configuration -@EntityScan(basePackages = {"uk.ac.ebi.eva.accession.core.persistence"}) -@EnableMongoRepositories(basePackages = { - "uk.ac.ebi.eva.accession.core.persistence", - "uk.ac.ebi.ampt2d.commons.accession.persistence.mongodb.repository"}) -@EnableMongoAuditing -@AutoConfigureDataMongo -public class MongoTestConfiguration { - - @Value("${spring.data.mongodb.uri}") - private String mongoUri; - - @Value("${spring.data.mongodb.database}") - private String database; - - @Value("${mongodb.read-preference:primary}") - private String readPreference; - - @Bean - @Primary - public MongoClient mongoClient() { - return MongoClients.create(mongoUri); - } - - @Bean("primaryFactory") - @Primary - public MongoDatabaseFactory mongoDbFactory() { - return new SimpleMongoClientDatabaseFactory(mongoClient(), database); - } - - @Bean - @Primary - public MongoCustomConversions mongoCustomConversions() { - return new MongoCustomConversions(Collections.emptyList()); - } - - @Bean - @Primary - public MongoMappingContext mongoMappingContext(MongoCustomConversions conversions) { - MongoMappingContext context = new MongoMappingContext(); - context.setSimpleTypeHolder(conversions.getSimpleTypeHolder()); - context.setAutoIndexCreation(false); - return context; - } - - @Bean - @Primary - public MongoTemplate mongoTemplate(MongoMappingContext mongoMappingContext, - MongoCustomConversions conversions) { - MappingMongoConverter converter = new MappingMongoConverter( - new DefaultDbRefResolver(mongoDbFactory()), mongoMappingContext); - converter.setCustomConversions(conversions); - converter.setTypeMapper(new DefaultMongoTypeMapper(null)); - converter.afterPropertiesSet(); - MongoTemplate mongoTemplate = new MongoTemplate(mongoDbFactory(), converter); - mongoTemplate.setReadPreference(ReadPreference.valueOf(readPreference)); - mongoTemplate.setWriteConcern(WriteConcern.MAJORITY); - mongoTemplate.setWriteResultChecking(WriteResultChecking.EXCEPTION); - return mongoTemplate; - } - -} \ No newline at end of file diff --git a/eva-accession-deprecate/src/test/resources/deprecate-submitted-variants-from-file-test.properties b/eva-accession-deprecate/src/test/resources/deprecate-submitted-variants-from-file-test.properties index 5ea0e7638..097a7e3c1 100644 --- a/eva-accession-deprecate/src/test/resources/deprecate-submitted-variants-from-file-test.properties +++ b/eva-accession-deprecate/src/test/resources/deprecate-submitted-variants-from-file-test.properties @@ -14,6 +14,11 @@ accessioning.instanceId=test-instance-01 accessioning.submitted.categoryId=test-ss accessioning.clustered.categoryId=test-rs +continuous.id.blocks.datasource.driver-class-name=org.hsqldb.jdbc.JDBCDriver +continuous.id.blocks.datasource.jdbc-url=jdbc:hsqldb:mem:db;sql.syntax_pgs=true;DB_CLOSE_DELAY=-1 +continuous.id.blocks.datasource.username=SA +continuous.id.blocks.datasource.password= + accessioning.monotonic.test-ss.blockSize=100000 accessioning.monotonic.test-ss.blockStartValue=4 accessioning.monotonic.test-ss.nextBlockInterval=1000000000 diff --git a/eva-accession-deprecate/src/test/resources/study-submitted-variants-test.properties b/eva-accession-deprecate/src/test/resources/study-submitted-variants-test.properties index e0e437959..c70a51775 100644 --- a/eva-accession-deprecate/src/test/resources/study-submitted-variants-test.properties +++ b/eva-accession-deprecate/src/test/resources/study-submitted-variants-test.properties @@ -11,6 +11,11 @@ accessioning.instanceId=test-instance-01 accessioning.submitted.categoryId=test-ss accessioning.clustered.categoryId=test-rs +continuous.id.blocks.datasource.driver-class-name=org.hsqldb.jdbc.JDBCDriver +continuous.id.blocks.datasource.jdbc-url=jdbc:hsqldb:mem:db;sql.syntax_pgs=true;DB_CLOSE_DELAY=-1 +continuous.id.blocks.datasource.username=SA +continuous.id.blocks.datasource.password= + accessioning.monotonic.test-ss.blockSize=100000 accessioning.monotonic.test-ss.blockStartValue=4 accessioning.monotonic.test-ss.nextBlockInterval=1000000000 diff --git a/eva-accession-import-dbsnp2/pom.xml b/eva-accession-import-dbsnp2/pom.xml index 03889334e..4e0670781 100644 --- a/eva-accession-import-dbsnp2/pom.xml +++ b/eva-accession-import-dbsnp2/pom.xml @@ -44,6 +44,11 @@ spring-batch-test test + + org.hsqldb + hsqldb + test + org.springframework.boot spring-boot-testcontainers @@ -72,11 +77,6 @@ uk.ac.ebi.eva variation-commons-batch - - com.h2database - h2 - runtime - diff --git a/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/Application.java b/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/Application.java index f5e225871..350ea8044 100644 --- a/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/Application.java +++ b/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/Application.java @@ -17,9 +17,23 @@ import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.autoconfigure.batch.BatchAutoConfiguration; +import org.springframework.boot.autoconfigure.data.mongo.MongoDataAutoConfiguration; +import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration; +import org.springframework.boot.autoconfigure.jdbc.DataSourceTransactionManagerAutoConfiguration; +import org.springframework.boot.autoconfigure.orm.jpa.HibernateJpaAutoConfiguration; import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.context.annotation.Import; +import uk.ac.ebi.eva.accession.core.configuration.InMemoryBatchConfiguration; -@SpringBootApplication +@SpringBootApplication(exclude = { + MongoDataAutoConfiguration.class, + DataSourceAutoConfiguration.class, + DataSourceTransactionManagerAutoConfiguration.class, + HibernateJpaAutoConfiguration.class, + BatchAutoConfiguration.class +}) +@Import(InMemoryBatchConfiguration.class) public class Application { public static void main(String[] args) { diff --git a/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/configuration/batch/jobs/ImportDbsnpJsonVariantsJobConfiguration.java b/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/configuration/batch/jobs/ImportDbsnpJsonVariantsJobConfiguration.java index d747cfa86..c28200fd8 100644 --- a/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/configuration/batch/jobs/ImportDbsnpJsonVariantsJobConfiguration.java +++ b/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/configuration/batch/jobs/ImportDbsnpJsonVariantsJobConfiguration.java @@ -16,10 +16,8 @@ package uk.ac.ebi.eva.accession.dbsnp2.configuration.batch.jobs; import org.springframework.batch.core.Job; -import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.batch.core.job.builder.JobBuilder; import org.springframework.batch.core.job.flow.Flow; -import org.springframework.batch.core.launch.support.RunIdIncrementer; import org.springframework.batch.core.repository.JobRepository; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; @@ -33,7 +31,6 @@ * Configuration to run a job that imports a dbSNP JSON file. */ @Configuration -@EnableBatchProcessing public class ImportDbsnpJsonVariantsJobConfiguration { @Autowired @@ -43,7 +40,6 @@ public class ImportDbsnpJsonVariantsJobConfiguration { @Bean(IMPORT_DBSNP_JSON_VARIANTS_JOB) public Job importDbsnpJsonVariantsJob(JobRepository jobRepository) { return new JobBuilder(IMPORT_DBSNP_JSON_VARIANTS_JOB, jobRepository) - .incrementer(new RunIdIncrementer()) .start(importFlow) .end() .build(); diff --git a/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/configuration/batch/steps/ImportDbsnpJsonVariantsStepConfiguration.java b/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/configuration/batch/steps/ImportDbsnpJsonVariantsStepConfiguration.java index 0828b2229..501c10e33 100644 --- a/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/configuration/batch/steps/ImportDbsnpJsonVariantsStepConfiguration.java +++ b/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/configuration/batch/steps/ImportDbsnpJsonVariantsStepConfiguration.java @@ -18,7 +18,6 @@ import com.fasterxml.jackson.databind.JsonNode; import org.springframework.batch.core.Step; import org.springframework.batch.core.StepExecutionListener; -import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.batch.core.repository.JobRepository; import org.springframework.batch.core.step.builder.StepBuilder; import org.springframework.batch.item.ItemProcessor; @@ -32,6 +31,7 @@ import org.springframework.transaction.PlatformTransactionManager; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpClusteredVariantEntity; +import static uk.ac.ebi.eva.accession.core.configuration.InMemoryBatchConfiguration.BATCH_TRANSACTION_MANAGER; import static uk.ac.ebi.eva.accession.dbsnp2.configuration.BeanNames.DBSNP_JSON_VARIANT_PROCESSOR; import static uk.ac.ebi.eva.accession.dbsnp2.configuration.BeanNames.DBSNP_JSON_VARIANT_READER; import static uk.ac.ebi.eva.accession.dbsnp2.configuration.BeanNames.DBSNP_JSON_VARIANT_WRITER; @@ -42,7 +42,6 @@ * Configuration for dbSNP JSON import flow step */ @Configuration -@EnableBatchProcessing public class ImportDbsnpJsonVariantsStepConfiguration { @Autowired @@ -63,7 +62,8 @@ public class ImportDbsnpJsonVariantsStepConfiguration { @Bean(IMPORT_DBSNP_JSON_VARIANTS_STEP) - public Step importDbsnpJsonVariantsStep(JobRepository jobRepository, PlatformTransactionManager transactionManager, + public Step importDbsnpJsonVariantsStep(JobRepository jobRepository, + @Qualifier(BATCH_TRANSACTION_MANAGER) PlatformTransactionManager transactionManager, SimpleCompletionPolicy chunkSizeCompletionPolicy) { return new StepBuilder(IMPORT_DBSNP_JSON_VARIANTS_STEP, jobRepository) .chunk(chunkSizeCompletionPolicy, transactionManager) diff --git a/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/parameters/InputParameters.java b/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/parameters/InputParameters.java index 641367832..53d70ef16 100644 --- a/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/parameters/InputParameters.java +++ b/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/parameters/InputParameters.java @@ -27,16 +27,15 @@ public class InputParameters { private int previousImportedBuild; private boolean incrementalImport; private int chunkSize; - private boolean forceRestart; private boolean forceImport; public JobParameters toJobParameters() { return new JobParametersBuilder() - .addString("input", input) - .addString("genbankAssembly", genbankAssembly) - .addString("refseqAssembly", refseqAssembly) - .addLong("chunkSize", (long) chunkSize, false) - .toJobParameters(); + .addString("input", input) + .addString("genbankAssembly", genbankAssembly) + .addString("refseqAssembly", refseqAssembly) + .addLong("chunkSize", (long) chunkSize, false) + .toJobParameters(); } public String getInput() { @@ -95,14 +94,6 @@ public void setChunkSize(int chunkSize) { this.chunkSize = chunkSize; } - public boolean isForceRestart() { - return forceRestart; - } - - public void setForceRestart(boolean forceRestart) { - this.forceRestart = forceRestart; - } - public boolean isForceImport() { return forceImport; } diff --git a/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/runner/DbsnpJsonImportVariantsJobLauncherCommandLineRunner.java b/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/runner/DbsnpJsonImportVariantsJobLauncherCommandLineRunner.java index ff80ea812..e297d6770 100644 --- a/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/runner/DbsnpJsonImportVariantsJobLauncherCommandLineRunner.java +++ b/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/runner/DbsnpJsonImportVariantsJobLauncherCommandLineRunner.java @@ -18,7 +18,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.batch.core.Job; -import org.springframework.batch.core.JobExecution; import org.springframework.batch.core.JobExecutionException; import org.springframework.batch.core.JobParameters; import org.springframework.batch.core.JobParametersInvalidException; @@ -35,11 +34,9 @@ import org.springframework.context.ApplicationEventPublisherAware; import org.springframework.stereotype.Component; import org.springframework.util.PatternMatchUtils; -import uk.ac.ebi.eva.accession.core.runner.CommandLineRunnerUtils; import uk.ac.ebi.eva.accession.dbsnp2.parameters.InputParameters; import uk.ac.ebi.eva.commons.batch.exception.NoJobToExecuteException; import uk.ac.ebi.eva.commons.batch.exception.NoParametersHaveBeenPassedException; -import uk.ac.ebi.eva.commons.batch.exception.NoPreviousJobExecutionException; import uk.ac.ebi.eva.commons.batch.job.JobExecutionApplicationListener; import uk.ac.ebi.eva.commons.batch.job.JobStatusManager; @@ -115,17 +112,10 @@ public void run(String... args) throws JobExecutionException { JobStatusManager.checkIfJobNameHasBeenDefined(jobName); JobStatusManager.checkIfPropertiesHaveBeenProvided(jobParameters); - if (inputParameters.isForceRestart()) { - JobExecution previousJobExecution = CommandLineRunnerUtils.getLastJobExecution(jobName, jobExplorer, - jobParameters); - CommandLineRunnerUtils.markPreviousJobAsFailed(jobName, jobRepository, - previousJobExecution != null ? previousJobExecution.getJobParameters() : jobParameters); - } else { - jobParameters = CommandLineRunnerUtils.addRunIDToJobParameters(jobName, jobExplorer, jobParameters); - } + launchJob(jobParameters); - } catch (NoPreviousJobExecutionException | NoParametersHaveBeenPassedException | NoJobToExecuteException - | JobParametersInvalidException | JobExecutionAlreadyRunningException e) { + } catch (NoParametersHaveBeenPassedException | NoJobToExecuteException | JobParametersInvalidException + | JobExecutionAlreadyRunningException e) { logger.error(e.getMessage()); logger.debug("Error trace", e); abnormalExit = true; diff --git a/eva-accession-import-dbsnp2/src/main/resources/application.properties b/eva-accession-import-dbsnp2/src/main/resources/application.properties index 2730fabe3..f4b6ab7dc 100644 --- a/eva-accession-import-dbsnp2/src/main/resources/application.properties +++ b/eva-accession-import-dbsnp2/src/main/resources/application.properties @@ -10,7 +10,6 @@ parameters.assemblyReportUrl= parameters.previousImportedBuild= parameters.incrementalImport=false parameters.chunkSize= -parameters.forceRestart=false # MongoDB for storing imported accessions spring.data.mongodb.host= diff --git a/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/batch/io/DbsnpJsonClusteredVariantsWriterTest.java b/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/batch/io/DbsnpJsonClusteredVariantsWriterTest.java index 8c2b413dd..686a52204 100644 --- a/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/batch/io/DbsnpJsonClusteredVariantsWriterTest.java +++ b/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/batch/io/DbsnpJsonClusteredVariantsWriterTest.java @@ -41,9 +41,9 @@ import uk.ac.ebi.eva.accession.core.repository.nonhuman.dbsnp.DbsnpClusteredVariantAccessioningRepository; import uk.ac.ebi.eva.accession.core.repository.nonhuman.dbsnp.DbsnpClusteredVariantOperationRepository; import uk.ac.ebi.eva.accession.core.summary.ClusteredVariantSummaryFunction; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.dbsnp2.test.BatchTestConfiguration; -import uk.ac.ebi.eva.accession.dbsnp2.test.MongoTestConfiguration; import uk.ac.ebi.eva.commons.core.models.VariantType; import java.time.LocalDateTime; diff --git a/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/configuration/jobs/ImportDbsnpJsonVariantsJobConfigurationTest.java b/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/configuration/jobs/ImportDbsnpJsonVariantsJobConfigurationTest.java index eb4fcded3..f1d3a1792 100644 --- a/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/configuration/jobs/ImportDbsnpJsonVariantsJobConfigurationTest.java +++ b/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/configuration/jobs/ImportDbsnpJsonVariantsJobConfigurationTest.java @@ -27,11 +27,10 @@ import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit.jupiter.SpringExtension; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.dbsnp2.parameters.InputParameters; -import uk.ac.ebi.eva.accession.dbsnp2.test.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.accession.dbsnp2.test.BatchTestConfiguration; -import uk.ac.ebi.eva.accession.dbsnp2.test.MongoTestConfiguration; import java.util.Collection; import java.util.Collections; @@ -43,8 +42,7 @@ import static uk.ac.ebi.eva.accession.dbsnp2.test.BatchTestConfiguration.JOB_IMPORT_DBSNP_JSON_VARIANTS_JOB; @ExtendWith(SpringExtension.class) -@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class, - BatchJobRepositoryTestConfiguration.class}) +@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class}) @TestPropertySource("classpath:application.properties") public class ImportDbsnpJsonVariantsJobConfigurationTest extends MongoTestContainerHelper { diff --git a/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/configuration/steps/ImportDbsnpJsonVariantsStepConfigurationTest.java b/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/configuration/steps/ImportDbsnpJsonVariantsStepConfigurationTest.java index 06522e338..666df1f93 100644 --- a/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/configuration/steps/ImportDbsnpJsonVariantsStepConfigurationTest.java +++ b/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/configuration/steps/ImportDbsnpJsonVariantsStepConfigurationTest.java @@ -30,18 +30,16 @@ import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpClusteredVariantEntity; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; -import uk.ac.ebi.eva.accession.dbsnp2.test.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.accession.dbsnp2.test.BatchTestConfiguration; -import uk.ac.ebi.eva.accession.dbsnp2.test.MongoTestConfiguration; import static org.junit.jupiter.api.Assertions.assertEquals; import static uk.ac.ebi.eva.accession.dbsnp2.configuration.BeanNames.IMPORT_DBSNP_JSON_VARIANTS_STEP; import static uk.ac.ebi.eva.accession.dbsnp2.test.BatchTestConfiguration.JOB_IMPORT_DBSNP_JSON_VARIANTS_JOB; @ExtendWith(SpringExtension.class) -@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class, - BatchJobRepositoryTestConfiguration.class}) +@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class}) @TestPropertySource("classpath:application.properties") public class ImportDbsnpJsonVariantsStepConfigurationTest extends MongoTestContainerHelper { diff --git a/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/runner/DbsnpJsonImportVariantsJobLauncherCommandLineRunnerTest.java b/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/runner/DbsnpJsonImportVariantsJobLauncherCommandLineRunnerTest.java index 3add32b31..611ee062c 100644 --- a/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/runner/DbsnpJsonImportVariantsJobLauncherCommandLineRunnerTest.java +++ b/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/runner/DbsnpJsonImportVariantsJobLauncherCommandLineRunnerTest.java @@ -22,8 +22,6 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; -import org.springframework.batch.core.JobInstance; -import org.springframework.batch.core.StepExecution; import org.springframework.batch.core.explore.JobExplorer; import org.springframework.batch.core.repository.JobRepository; import org.springframework.batch.item.file.FlatFileItemReader; @@ -34,13 +32,11 @@ import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit.jupiter.SpringExtension; -import uk.ac.ebi.eva.accession.core.runner.CommandLineRunnerUtils; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.dbsnp2.batch.io.BzipLazyResource; import uk.ac.ebi.eva.accession.dbsnp2.parameters.InputParameters; -import uk.ac.ebi.eva.accession.dbsnp2.test.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.accession.dbsnp2.test.BatchTestConfiguration; -import uk.ac.ebi.eva.accession.dbsnp2.test.MongoTestConfiguration; import uk.ac.ebi.eva.commons.core.utils.FileUtils; import java.io.BufferedReader; @@ -50,19 +46,14 @@ import java.io.InputStreamReader; import java.io.OutputStream; import java.nio.charset.StandardCharsets; -import java.util.Arrays; import java.util.Collections; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotEquals; import static uk.ac.ebi.eva.accession.dbsnp2.configuration.BeanNames.IMPORT_DBSNP_JSON_VARIANTS_JOB; -import static uk.ac.ebi.eva.accession.dbsnp2.configuration.BeanNames.IMPORT_DBSNP_JSON_VARIANTS_STEP; import static uk.ac.ebi.eva.accession.dbsnp2.runner.DbsnpJsonImportVariantsJobLauncherCommandLineRunner.EXIT_WITHOUT_ERRORS; -import static uk.ac.ebi.eva.accession.dbsnp2.runner.DbsnpJsonImportVariantsJobLauncherCommandLineRunner.EXIT_WITH_ERRORS; @ExtendWith(SpringExtension.class) -@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class, - BatchJobRepositoryTestConfiguration.class}) +@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class}) @TestPropertySource("classpath:application.properties") @SpringBatchTest public class DbsnpJsonImportVariantsJobLauncherCommandLineRunnerTest extends MongoTestContainerHelper { @@ -113,14 +104,12 @@ public void setUp() throws Exception { jobRepositoryTestUtils = new JobRepositoryTestUtils(jobRepository); runner.setJobName(IMPORT_DBSNP_JSON_VARIANTS_JOB); jobRepositoryTestUtils.removeJobExecutions(); - inputParameters.setForceRestart(false); useOriginalJsonFile(); } @AfterEach public void tearDown() { jobRepositoryTestUtils.removeJobExecutions(); - inputParameters.setForceRestart(false); } @Test @@ -137,142 +126,24 @@ public void restartCompletedJobThatIsAlreadyInTheRepository() throws Exception { runner.run(); assertEquals(EXIT_WITHOUT_ERRORS, runner.getExitCode()); - inputParameters.setForceRestart(true); runner.run(); assertEquals(EXIT_WITHOUT_ERRORS, runner.getExitCode()); } - @Test - @DirtiesContext - public void restartFailedJobThatIsAlreadyInTheRepository() throws Exception { - useTempJsonFile(); - injectErrorIntoTempJson(); - JobInstance failingJobInstance = runJobAandCheckResults(); - - inputParameters.setForceRestart(true); - remediateTempVcfError(); - runJobBAndCheckRestart(failingJobInstance); - } - - private JobInstance runJobAandCheckResults() throws Exception { - runner.run(); - assertEquals(EXIT_WITH_ERRORS, runner.getExitCode()); - JobInstance currentJobInstance = CommandLineRunnerUtils.getLastJobExecution(IMPORT_DBSNP_JSON_VARIANTS_JOB, - jobExplorer, - inputParameters.toJobParameters()) - .getJobInstance(); - StepExecution stepExecution = jobRepository.getLastStepExecution(currentJobInstance, - IMPORT_DBSNP_JSON_VARIANTS_STEP); - //Ensure that only the first batch was written (batch size is 2 and error was at line#4) - assertEquals(inputParameters.getChunkSize(), stepExecution.getWriteCount()); - - return currentJobInstance; - } - - private void runJobBAndCheckRestart(JobInstance failingJobInstance) throws Exception { - runner.run(); - assertEquals(EXIT_WITHOUT_ERRORS, runner.getExitCode()); - JobInstance currentJobInstance = CommandLineRunnerUtils.getLastJobExecution(IMPORT_DBSNP_JSON_VARIANTS_JOB, - jobExplorer, - inputParameters.toJobParameters()) - .getJobInstance(); - assertNotEquals(failingJobInstance.getInstanceId(), currentJobInstance.getInstanceId()); - } - @Test @DirtiesContext public void forceRestartButNoJobInTheRepository() throws Exception { - inputParameters.setForceRestart(true); assertEquals(Collections.EMPTY_LIST, jobExplorer.getJobNames()); runner.run(); assertEquals(EXIT_WITHOUT_ERRORS, runner.getExitCode()); } - @Test - @DirtiesContext - public void resumeFailingJobFromCorrectChunk() throws Exception { - // Jobs A, B, C are run chronological order; A and C have SAME parameters; - // A is the job that is run after VCF fault injection (as part of the runTestWithFaultInjection method), - // therefore should fail. - // B is a job run with the original VCF without any faults (run separately), therefore should succeed. - // C is a job with the same parameters as A run after VCF fault remediation (as part of the - // runTestWithFaultInjection method), therefore should resume A and succeed. - - useTempJsonFile(); - injectErrorIntoTempJson(); - JobInstance failingJobInstance = runJobAandCheckResults(); - - runJobBAndCheckResults(); - - remediateTempVcfError(); - runJobCAndCheckResumption(failingJobInstance); - } - - private void runJobBAndCheckResults() throws Exception { - useOriginalJsonFile(); - runner.run(); - assertEquals(EXIT_WITHOUT_ERRORS, runner.getExitCode()); - - //Restore state so that Job C can continue running after fault remediation - useTempJsonFile(); - } - - private void runJobCAndCheckResumption(JobInstance failingJobInstance) throws Exception { - runner.run(); - JobInstance currentJobInstance = CommandLineRunnerUtils.getLastJobExecution(IMPORT_DBSNP_JSON_VARIANTS_JOB, - jobExplorer, - inputParameters.toJobParameters()) - .getJobInstance(); - StepExecution stepExecution = jobRepository.getLastStepExecution(currentJobInstance, - IMPORT_DBSNP_JSON_VARIANTS_STEP); - // Did we resume the previous failed job instance? - assertEquals(failingJobInstance.getInstanceId(), currentJobInstance.getInstanceId()); - - int numberOfLinesInJson = getNumberOfLinesInJsonString(originalJsonContent); - // Test resumption point - did we pick up where we left off? - // Ensure all the batches other than the first batch were processed - assertEquals(numberOfLinesInJson - inputParameters.getChunkSize(), stepExecution.getWriteCount()); - assertEquals(EXIT_WITHOUT_ERRORS, runner.getExitCode()); - } - - private void injectErrorIntoTempJson() throws Exception { - // Intentionally inject error in entry#9 in the original Json - String modifiedJsonContent = originalJsonContent.replace("3069077", "3069077jibberish"); - writeToTempJsonFile(modifiedJsonContent); - } - - private void remediateTempVcfError() throws Exception { - writeToTempJsonFile(originalJsonContent); - } - private void useOriginalJsonFile() throws Exception { inputParameters.setInput(originalJsonInputFilePath); variantReader.setResource(FileUtils.getResource(new File(originalJsonInputFilePath))); } - private void useTempJsonFile() throws Exception { - // The following does not actually change the wiring of the variantReader since the wiring happens before the tests - // This setVcf is only to facilitate identifying jobs in the job repo by parameter - // (those that use original vs temp JSON) - inputParameters.setInput(tempJsonInputFileToTestFailingJobs.getAbsolutePath()); - /* - * Change the auto-wired JSON for variantReader at runtime - * Rationale: - * 1) Why not use two test configurations, one for a JSON that fails validation and another for a JSON - * that won't and test resumption? - * Beginning Spring Boot 2, job resumption can only happen when input parameters to the restarted job - * is the same as the failed job. - * Therefore, a test to check resumption cannot have two different config files with different - * parameters.input. - * This test therefore creates a dynamic JSON and injects errors at runtime to the JSON thus preserving - * the input parameter but changing the JSON content. - * 2) Why not artificially inject a variantReader exception? - * This will preclude us from verifying job resumption from a precise line in the JSON. - */ - variantReader.setResource(FileUtils.getResource(tempJsonInputFileToTestFailingJobs)); - } - private void writeToTempJsonFile(String modifiedJsonContent) throws IOException { OutputStream bzipOutputStream = new BzipLazyResource(tempJsonInputFileToTestFailingJobs.getAbsolutePath()).getOutputStream(); @@ -290,10 +161,4 @@ private String getOriginalJsonContent(String inputJsonPath) throws IOException { } return originalJsonContent.toString(); } - - private int getNumberOfLinesInJsonString(String jsonString) { - return (int) Arrays.stream(jsonString.split(System.lineSeparator())) - .filter(line -> !line.startsWith("#")) - .count(); - } } \ No newline at end of file diff --git a/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/test/BatchJobRepositoryTestConfiguration.java b/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/test/BatchJobRepositoryTestConfiguration.java deleted file mode 100644 index dd4a09fb2..000000000 --- a/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/test/BatchJobRepositoryTestConfiguration.java +++ /dev/null @@ -1,22 +0,0 @@ -package uk.ac.ebi.eva.accession.dbsnp2.test; - -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Primary; -import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseBuilder; -import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseType; - -import javax.sql.DataSource; - -@Configuration -public class BatchJobRepositoryTestConfiguration { - - @Bean - @Primary - public DataSource dataSource() { - return new EmbeddedDatabaseBuilder() - .setType(EmbeddedDatabaseType.HSQL) - .addScript("org/springframework/batch/core/schema-hsqldb.sql") - .build(); - } -} \ No newline at end of file diff --git a/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/test/BatchTestConfiguration.java b/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/test/BatchTestConfiguration.java index e88926287..2bfa8dc56 100644 --- a/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/test/BatchTestConfiguration.java +++ b/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/test/BatchTestConfiguration.java @@ -16,15 +16,14 @@ package uk.ac.ebi.eva.accession.dbsnp2.test; import org.springframework.batch.core.Job; -import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.batch.core.launch.JobLauncher; import org.springframework.batch.core.repository.JobRepository; import org.springframework.batch.test.JobLauncherTestUtils; import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Import; +import uk.ac.ebi.eva.accession.core.configuration.InMemoryBatchConfiguration; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.MongoConfiguration; import uk.ac.ebi.eva.accession.dbsnp2.configuration.InputParametersConfiguration; import uk.ac.ebi.eva.accession.dbsnp2.configuration.batch.flow.ImportDbsnpJsonFlowConfiguration; @@ -41,9 +40,8 @@ import static uk.ac.ebi.eva.accession.dbsnp2.configuration.BeanNames.IMPORT_DBSNP_JSON_VARIANTS_JOB; @Configuration -@EnableAutoConfiguration -@EnableBatchProcessing -@Import({ListenersConfiguration.class, +@Import({InMemoryBatchConfiguration.class, + ListenersConfiguration.class, MongoConfiguration.class, ImportDbsnpJsonVariantsJobConfiguration.class, ImportDbsnpJsonVariantsStepConfiguration.class, diff --git a/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/test/MongoTestConfiguration.java b/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/test/MongoTestConfiguration.java deleted file mode 100644 index 63280b6dd..000000000 --- a/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/test/MongoTestConfiguration.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Copyright 2021 EMBL - European Bioinformatics Institute - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package uk.ac.ebi.eva.accession.dbsnp2.test; - -import com.mongodb.ReadPreference; -import com.mongodb.WriteConcern; -import com.mongodb.client.MongoClient; -import com.mongodb.client.MongoClients; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.domain.EntityScan; -import org.springframework.boot.test.autoconfigure.data.mongo.AutoConfigureDataMongo; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Primary; -import org.springframework.data.mongodb.MongoDatabaseFactory; -import org.springframework.data.mongodb.config.EnableMongoAuditing; -import org.springframework.data.mongodb.core.MongoTemplate; -import org.springframework.data.mongodb.core.SimpleMongoClientDatabaseFactory; -import org.springframework.data.mongodb.core.WriteResultChecking; -import org.springframework.data.mongodb.core.convert.DefaultDbRefResolver; -import org.springframework.data.mongodb.core.convert.DefaultMongoTypeMapper; -import org.springframework.data.mongodb.core.convert.MappingMongoConverter; -import org.springframework.data.mongodb.core.convert.MongoCustomConversions; -import org.springframework.data.mongodb.core.mapping.MongoMappingContext; -import org.springframework.data.mongodb.repository.config.EnableMongoRepositories; - -import java.util.Collections; - -@Configuration -@EntityScan(basePackages = {"uk.ac.ebi.eva.accession.core.persistence"}) -@EnableMongoRepositories(basePackages = { - "uk.ac.ebi.eva.accession.core.persistence", - "uk.ac.ebi.ampt2d.commons.accession.persistence.mongodb.repository"}) -@EnableMongoAuditing -@AutoConfigureDataMongo -public class MongoTestConfiguration { - - @Value("${spring.data.mongodb.uri}") - private String mongoUri; - - @Value("${spring.data.mongodb.database}") - private String database; - - @Value("${mongodb.read-preference:primary}") - private String readPreference; - - @Bean - @Primary - public MongoClient mongoClient() { - return MongoClients.create(mongoUri); - } - - @Bean("primaryFactory") - @Primary - public MongoDatabaseFactory mongoDbFactory() { - return new SimpleMongoClientDatabaseFactory(mongoClient(), database); - } - - @Bean - @Primary - public MongoCustomConversions mongoCustomConversions() { - return new MongoCustomConversions(Collections.emptyList()); - } - - @Bean - @Primary - public MongoMappingContext mongoMappingContext(MongoCustomConversions conversions) { - MongoMappingContext context = new MongoMappingContext(); - context.setSimpleTypeHolder(conversions.getSimpleTypeHolder()); - context.setAutoIndexCreation(false); - return context; - } - - @Bean - @Primary - public MongoTemplate mongoTemplate(MongoMappingContext mongoMappingContext, - MongoCustomConversions conversions) { - MappingMongoConverter converter = new MappingMongoConverter( - new DefaultDbRefResolver(mongoDbFactory()), mongoMappingContext); - converter.setCustomConversions(conversions); - converter.setTypeMapper(new DefaultMongoTypeMapper(null)); - converter.afterPropertiesSet(); - MongoTemplate mongoTemplate = new MongoTemplate(mongoDbFactory(), converter); - mongoTemplate.setReadPreference(ReadPreference.valueOf(readPreference)); - mongoTemplate.setWriteConcern(WriteConcern.MAJORITY); - mongoTemplate.setWriteResultChecking(WriteResultChecking.EXCEPTION); - return mongoTemplate; - } -} diff --git a/eva-accession-import-dbsnp2/src/test/resources/application.properties b/eva-accession-import-dbsnp2/src/test/resources/application.properties index d2e66060f..c4197edc6 100644 --- a/eva-accession-import-dbsnp2/src/test/resources/application.properties +++ b/eva-accession-import-dbsnp2/src/test/resources/application.properties @@ -10,7 +10,6 @@ parameters.assemblyReportUrl=file:src/test/resources/input-files/GCF_000001405.3 parameters.previousImportedBuild=10 parameters.incrementalImport=true parameters.chunkSize=5 -parameters.forceRestart=false # MongoDB for storing imported accessions spring.data.mongodb.database=admin diff --git a/eva-accession-pipeline/pom.xml b/eva-accession-pipeline/pom.xml index 34e3f8e38..750a7edd9 100644 --- a/eva-accession-pipeline/pom.xml +++ b/eva-accession-pipeline/pom.xml @@ -45,6 +45,11 @@ test + + org.hsqldb + hsqldb + test + org.springframework.batch spring-batch-test diff --git a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/Application.java b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/Application.java index c707ac126..ccdafb238 100644 --- a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/Application.java +++ b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/Application.java @@ -15,14 +15,26 @@ */ package uk.ac.ebi.eva.accession.pipeline; -import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.autoconfigure.batch.BatchAutoConfiguration; +import org.springframework.boot.autoconfigure.data.mongo.MongoDataAutoConfiguration; +import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration; +import org.springframework.boot.autoconfigure.jdbc.DataSourceTransactionManagerAutoConfiguration; +import org.springframework.boot.autoconfigure.orm.jpa.HibernateJpaAutoConfiguration; import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.context.annotation.Import; +import uk.ac.ebi.eva.accession.core.configuration.ContiguousIdBlocksDataSourceConfiguration; +import uk.ac.ebi.eva.accession.core.configuration.InMemoryBatchConfiguration; -// TODO: eva-pipeline is also excluding MongoDataAutoConfiguration -@EnableBatchProcessing -@SpringBootApplication +@SpringBootApplication(exclude = { + MongoDataAutoConfiguration.class, + DataSourceAutoConfiguration.class, + DataSourceTransactionManagerAutoConfiguration.class, + HibernateJpaAutoConfiguration.class, + BatchAutoConfiguration.class +}) +@Import({InMemoryBatchConfiguration.class, ContiguousIdBlocksDataSourceConfiguration.class}) public class Application { public static void main(String[] args) { diff --git a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/DuplicateSSAccQCJobConfiguration.java b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/DuplicateSSAccQCJobConfiguration.java index b3f570c39..185f15e4d 100644 --- a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/DuplicateSSAccQCJobConfiguration.java +++ b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/DuplicateSSAccQCJobConfiguration.java @@ -2,9 +2,7 @@ import org.springframework.batch.core.Job; import org.springframework.batch.core.Step; -import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.batch.core.job.builder.JobBuilder; -import org.springframework.batch.core.launch.support.RunIdIncrementer; import org.springframework.batch.core.repository.JobRepository; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; @@ -15,7 +13,6 @@ import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.DUPLICATE_SS_ACC_QC_STEP; @Configuration -@EnableBatchProcessing public class DuplicateSSAccQCJobConfiguration { @Autowired @@ -25,7 +22,6 @@ public class DuplicateSSAccQCJobConfiguration { @Bean(DUPLICATE_SS_ACC_QC_JOB) public Job duplicateSSAccQCJob(JobRepository jobRepository) { return new JobBuilder(DUPLICATE_SS_ACC_QC_JOB, jobRepository) - .incrementer(new RunIdIncrementer()) .start(duplicateSSAccQCStep) .build(); } diff --git a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/QCSubsnpAccessionsJobConfiguration.java b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/QCSubsnpAccessionsJobConfiguration.java index 6ee75487b..d85368080 100644 --- a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/QCSubsnpAccessionsJobConfiguration.java +++ b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/QCSubsnpAccessionsJobConfiguration.java @@ -2,9 +2,7 @@ import org.springframework.batch.core.Job; import org.springframework.batch.core.Step; -import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.batch.core.job.builder.JobBuilder; -import org.springframework.batch.core.launch.support.RunIdIncrementer; import org.springframework.batch.core.repository.JobRepository; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; @@ -15,7 +13,6 @@ import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.QC_SUBSNP_ACCESSION_STEP; @Configuration -@EnableBatchProcessing public class QCSubsnpAccessionsJobConfiguration { @Autowired @Qualifier(QC_SUBSNP_ACCESSION_STEP) @@ -24,7 +21,6 @@ public class QCSubsnpAccessionsJobConfiguration { @Bean(QC_SUBSNP_ACCESSION_JOB) public Job qcSubsnpAccessionJob(JobRepository jobRepository) { return new JobBuilder(QC_SUBSNP_ACCESSION_JOB, jobRepository) - .incrementer(new RunIdIncrementer()) .start(qcSubsnpAccessionStep) .build(); } diff --git a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/SSAccessionRecoveryJobConfiguration.java b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/SSAccessionRecoveryJobConfiguration.java index 878ed8ad9..5c008d250 100644 --- a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/SSAccessionRecoveryJobConfiguration.java +++ b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/SSAccessionRecoveryJobConfiguration.java @@ -3,9 +3,7 @@ import org.springframework.batch.core.Job; import org.springframework.batch.core.JobExecutionListener; import org.springframework.batch.core.Step; -import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.batch.core.job.builder.JobBuilder; -import org.springframework.batch.core.launch.support.RunIdIncrementer; import org.springframework.batch.core.repository.JobRepository; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; @@ -17,7 +15,6 @@ import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.SS_ACCESSION_RECOVERY_STEP; @Configuration -@EnableBatchProcessing public class SSAccessionRecoveryJobConfiguration { @Autowired @@ -31,7 +28,6 @@ public class SSAccessionRecoveryJobConfiguration { @Bean(SS_ACCESSION_RECOVERY_JOB) public Job createSSAccessionRecoveryJob(JobRepository jobRepository) { return new JobBuilder(SS_ACCESSION_RECOVERY_JOB, jobRepository) - .incrementer(new RunIdIncrementer()) .start(ssAccessionRecoveryStep) .listener(ssAccessionRecoveryJobListener) .build(); diff --git a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/SubsnpAccessionsJobConfiguration.java b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/SubsnpAccessionsJobConfiguration.java index 43290594d..c65602ce5 100644 --- a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/SubsnpAccessionsJobConfiguration.java +++ b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/SubsnpAccessionsJobConfiguration.java @@ -19,9 +19,7 @@ import org.springframework.batch.core.Job; import org.springframework.batch.core.JobExecutionListener; import org.springframework.batch.core.Step; -import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.batch.core.job.builder.JobBuilder; -import org.springframework.batch.core.launch.support.RunIdIncrementer; import org.springframework.batch.core.repository.JobRepository; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; @@ -35,7 +33,6 @@ import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.SUBSNP_ACCESSION_STEP; @Configuration -@EnableBatchProcessing public class SubsnpAccessionsJobConfiguration { @Autowired @@ -57,7 +54,6 @@ public class SubsnpAccessionsJobConfiguration { @Bean(SUBSNP_ACCESSION_JOB) public Job subsnpAccessionJob(JobRepository jobRepository) { return new JobBuilder(SUBSNP_ACCESSION_JOB, jobRepository) - .incrementer(new RunIdIncrementer()) .start(subsnpAccessionStep) .next(accessioningShutdownStep) .next(buildReportStep) diff --git a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/AccessioningShutdownStepConfiguration.java b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/AccessioningShutdownStepConfiguration.java index ccad6721b..55be0d87e 100644 --- a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/AccessioningShutdownStepConfiguration.java +++ b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/AccessioningShutdownStepConfiguration.java @@ -1,25 +1,26 @@ package uk.ac.ebi.eva.accession.pipeline.configuration.batch.steps; import org.springframework.batch.core.Step; -import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.batch.core.repository.JobRepository; import org.springframework.batch.core.step.builder.StepBuilder; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.transaction.PlatformTransactionManager; import uk.ac.ebi.eva.accession.core.service.nonhuman.SubmittedVariantAccessioningService; +import static uk.ac.ebi.eva.accession.core.configuration.InMemoryBatchConfiguration.BATCH_TRANSACTION_MANAGER; import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.ACCESSIONING_SHUTDOWN_STEP; @Configuration -@EnableBatchProcessing public class AccessioningShutdownStepConfiguration { @Autowired private SubmittedVariantAccessioningService submittedVariantAccessioningService; @Bean(ACCESSIONING_SHUTDOWN_STEP) - public Step accessioningShutDownStep(JobRepository jobRepository, PlatformTransactionManager transactionManager) { + public Step accessioningShutDownStep(JobRepository jobRepository, + @Qualifier(BATCH_TRANSACTION_MANAGER) PlatformTransactionManager transactionManager) { return new StepBuilder(ACCESSIONING_SHUTDOWN_STEP, jobRepository) .tasklet((contribution, chunkContext) -> { submittedVariantAccessioningService.shutDownAccessionGenerator(); diff --git a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/BuildReportStepConfiguration.java b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/BuildReportStepConfiguration.java index e7fe6d462..0855df0bc 100644 --- a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/BuildReportStepConfiguration.java +++ b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/BuildReportStepConfiguration.java @@ -17,11 +17,11 @@ package uk.ac.ebi.eva.accession.pipeline.configuration.batch.steps; import org.springframework.batch.core.Step; -import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.batch.core.repository.JobRepository; import org.springframework.batch.core.step.builder.StepBuilder; import org.springframework.batch.core.step.tasklet.TaskletStep; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.transaction.PlatformTransactionManager; @@ -30,17 +30,18 @@ import java.io.File; +import static uk.ac.ebi.eva.accession.core.configuration.InMemoryBatchConfiguration.BATCH_TRANSACTION_MANAGER; import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.BUILD_REPORT_STEP; @Configuration -@EnableBatchProcessing public class BuildReportStepConfiguration { @Autowired private InputParameters inputParameters; @Bean(BUILD_REPORT_STEP) - public Step buildReportStep(JobRepository jobRepository, PlatformTransactionManager transactionManager) { + public Step buildReportStep(JobRepository jobRepository, + @Qualifier(BATCH_TRANSACTION_MANAGER) PlatformTransactionManager transactionManager) { BuildReportTasklet tasklet = new BuildReportTasklet(new File(inputParameters.getOutputVcf())); TaskletStep step = new StepBuilder(BUILD_REPORT_STEP, jobRepository) .tasklet(tasklet, transactionManager) diff --git a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/DuplicateSSAccQCStepConfiguration.java b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/DuplicateSSAccQCStepConfiguration.java index 75a9cabde..26e596671 100644 --- a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/DuplicateSSAccQCStepConfiguration.java +++ b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/DuplicateSSAccQCStepConfiguration.java @@ -1,7 +1,6 @@ package uk.ac.ebi.eva.accession.pipeline.configuration.batch.steps; import org.springframework.batch.core.Step; -import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.batch.core.repository.JobRepository; import org.springframework.batch.core.step.builder.StepBuilder; import org.springframework.batch.core.step.tasklet.TaskletStep; @@ -17,13 +16,13 @@ import java.util.List; +import static uk.ac.ebi.eva.accession.core.configuration.InMemoryBatchConfiguration.BATCH_TRANSACTION_MANAGER; import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.DUPLICATE_SS_ACC_QC_PROCESSOR; import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.DUPLICATE_SS_ACC_QC_STEP; import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.DUPLICATE_SS_ACC_QC_WRITER; import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.SS_ACC_FILE_READER; @Configuration -@EnableBatchProcessing public class DuplicateSSAccQCStepConfiguration { @Autowired @@ -39,7 +38,8 @@ public class DuplicateSSAccQCStepConfiguration { private ItemWriter> duplicateSSAccQCWriter; @Bean(DUPLICATE_SS_ACC_QC_STEP) - public Step duplicateSSAccQCStep(JobRepository jobRepository, PlatformTransactionManager transactionManager) { + public Step duplicateSSAccQCStep(JobRepository jobRepository, + @Qualifier(BATCH_TRANSACTION_MANAGER) PlatformTransactionManager transactionManager) { TaskletStep step = new StepBuilder(DUPLICATE_SS_ACC_QC_STEP, jobRepository) // hardcoded the chunk size as 1, as the reader takes care of accumulating // and sending the chunk size (defined in properties file) elements to the processor diff --git a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/QCSubsnpAccessionsStepConfiguration.java b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/QCSubsnpAccessionsStepConfiguration.java index 6fe42f886..158fc2e83 100644 --- a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/QCSubsnpAccessionsStepConfiguration.java +++ b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/QCSubsnpAccessionsStepConfiguration.java @@ -17,7 +17,6 @@ package uk.ac.ebi.eva.accession.pipeline.configuration.batch.steps; import org.springframework.batch.core.Step; -import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.batch.core.repository.JobRepository; import org.springframework.batch.core.step.builder.StepBuilder; import org.springframework.batch.core.step.tasklet.TaskletStep; @@ -38,12 +37,12 @@ import java.io.File; import java.io.IOException; +import static uk.ac.ebi.eva.accession.core.configuration.InMemoryBatchConfiguration.BATCH_TRANSACTION_MANAGER; import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.QC_SUBSNP_ACCESSION_STEP; import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.REPORT_READER; import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.VARIANT_READER; @Configuration -@EnableBatchProcessing public class QCSubsnpAccessionsStepConfiguration { @Autowired @@ -64,7 +63,8 @@ public ItemStreamReader reportReader() throws IOException { } @Bean(QC_SUBSNP_ACCESSION_STEP) - public Step qcSubsnpAccessionStep(JobRepository jobRepository, PlatformTransactionManager transactionManager) throws IOException { + public Step qcSubsnpAccessionStep(JobRepository jobRepository, + @Qualifier(BATCH_TRANSACTION_MANAGER) PlatformTransactionManager transactionManager) throws IOException { ReportCheckTasklet tasklet = new ReportCheckTasklet(inputReader, reportReader(), inputParameters.getChunkSize() * 2, contigMapping); TaskletStep step = new StepBuilder(QC_SUBSNP_ACCESSION_STEP, jobRepository) diff --git a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/SSAccessionRecoveryStepConfiguration.java b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/SSAccessionRecoveryStepConfiguration.java index a87432c36..a17ae3d03 100644 --- a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/SSAccessionRecoveryStepConfiguration.java +++ b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/SSAccessionRecoveryStepConfiguration.java @@ -1,7 +1,6 @@ package uk.ac.ebi.eva.accession.pipeline.configuration.batch.steps; import org.springframework.batch.core.Step; -import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.batch.core.repository.JobRepository; import org.springframework.batch.core.step.builder.StepBuilder; import org.springframework.beans.factory.annotation.Autowired; @@ -15,7 +14,6 @@ import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.SS_ACCESSION_RECOVERY_STEP; @Configuration -@EnableBatchProcessing public class SSAccessionRecoveryStepConfiguration { @Autowired @Qualifier(SS_ACCESSION_RECOVERY_SERVICE) diff --git a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/SubsnpAccessionsStepConfiguration.java b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/SubsnpAccessionsStepConfiguration.java index 8f4902761..9e2c2db94 100644 --- a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/SubsnpAccessionsStepConfiguration.java +++ b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/SubsnpAccessionsStepConfiguration.java @@ -18,7 +18,6 @@ import org.springframework.batch.core.Step; import org.springframework.batch.core.StepExecutionListener; -import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.batch.core.repository.JobRepository; import org.springframework.batch.core.step.builder.StepBuilder; import org.springframework.batch.core.step.tasklet.TaskletStep; @@ -37,6 +36,7 @@ import uk.ac.ebi.eva.commons.core.models.IVariant; import uk.ac.ebi.eva.commons.core.models.pipeline.Variant; +import static uk.ac.ebi.eva.accession.core.configuration.InMemoryBatchConfiguration.BATCH_TRANSACTION_MANAGER; import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.ACCESSION_WRITER; import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.COMPOSITE_VARIANT_PROCESSOR; import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.SUBSNP_ACCESSION_STEP; @@ -44,7 +44,6 @@ import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.VARIANT_READER; @Configuration -@EnableBatchProcessing public class SubsnpAccessionsStepConfiguration { @Autowired @@ -67,7 +66,8 @@ public class SubsnpAccessionsStepConfiguration { private InvalidVariantSkipPolicy invalidVariantSkipPolicy; @Bean(SUBSNP_ACCESSION_STEP) - public Step subsnpAccessionStep(JobRepository jobRepository, PlatformTransactionManager transactionManager, + public Step subsnpAccessionStep(JobRepository jobRepository, + @Qualifier(BATCH_TRANSACTION_MANAGER) PlatformTransactionManager transactionManager, SimpleCompletionPolicy chunkSizeCompletionPolicy) { TaskletStep step = new StepBuilder(SUBSNP_ACCESSION_STEP, jobRepository) .chunk(chunkSizeCompletionPolicy, transactionManager) diff --git a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/parameters/InputParameters.java b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/parameters/InputParameters.java index 555481c7c..1f6e017e0 100644 --- a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/parameters/InputParameters.java +++ b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/parameters/InputParameters.java @@ -42,8 +42,6 @@ public class InputParameters { private int chunkSize; - private boolean forceRestart; - private ContigNaming contigNaming; private String duplicateSSAccFile; @@ -143,14 +141,6 @@ public void setChunkSize(int chunkSize) { this.chunkSize = chunkSize; } - public boolean isForceRestart() { - return forceRestart; - } - - public void setForceRestart(boolean forceRestart) { - this.forceRestart = forceRestart; - } - public ContigNaming getContigNaming() { return contigNaming; } diff --git a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/runner/EvaAccessionJobLauncherCommandLineRunner.java b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/runner/EvaAccessionJobLauncherCommandLineRunner.java index adcc1202c..da28d1609 100644 --- a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/runner/EvaAccessionJobLauncherCommandLineRunner.java +++ b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/runner/EvaAccessionJobLauncherCommandLineRunner.java @@ -18,7 +18,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.batch.core.Job; -import org.springframework.batch.core.JobExecution; import org.springframework.batch.core.JobExecutionException; import org.springframework.batch.core.JobParameters; import org.springframework.batch.core.JobParametersInvalidException; @@ -35,11 +34,9 @@ import org.springframework.context.ApplicationEventPublisherAware; import org.springframework.stereotype.Component; import org.springframework.util.PatternMatchUtils; -import uk.ac.ebi.eva.accession.core.runner.CommandLineRunnerUtils; import uk.ac.ebi.eva.accession.pipeline.parameters.InputParameters; import uk.ac.ebi.eva.commons.batch.exception.NoJobToExecuteException; import uk.ac.ebi.eva.commons.batch.exception.NoParametersHaveBeenPassedException; -import uk.ac.ebi.eva.commons.batch.exception.NoPreviousJobExecutionException; import uk.ac.ebi.eva.commons.batch.exception.UnknownJobException; import uk.ac.ebi.eva.commons.batch.job.JobExecutionApplicationListener; import uk.ac.ebi.eva.commons.batch.job.JobStatusManager; @@ -120,17 +117,10 @@ public void run(String... args) throws JobExecutionException { JobStatusManager.checkIfJobNameHasBeenDefined(jobName); JobStatusManager.checkIfPropertiesHaveBeenProvided(jobParameters); - if (inputParameters.isForceRestart()) { - JobExecution previousJobExecution = CommandLineRunnerUtils.getLastJobExecution(jobName, jobExplorer, - jobParameters); - CommandLineRunnerUtils.markPreviousJobAsFailed(jobName, jobRepository, - previousJobExecution != null ? previousJobExecution.getJobParameters() : jobParameters); - } else { - jobParameters = CommandLineRunnerUtils.addRunIDToJobParameters(jobName, jobExplorer, jobParameters); - } + launchJob(jobParameters); - } catch (NoJobToExecuteException | NoParametersHaveBeenPassedException | NoPreviousJobExecutionException - | UnknownJobException | JobParametersInvalidException | JobExecutionAlreadyRunningException e) { + } catch (NoJobToExecuteException | NoParametersHaveBeenPassedException | UnknownJobException + | JobParametersInvalidException | JobExecutionAlreadyRunningException e) { logger.error(e.getMessage()); logger.debug("Error trace", e); abnormalExit = true; diff --git a/eva-accession-pipeline/src/main/resources/application.properties b/eva-accession-pipeline/src/main/resources/application.properties index 2b04016ec..135646cf8 100644 --- a/eva-accession-pipeline/src/main/resources/application.properties +++ b/eva-accession-pipeline/src/main/resources/application.properties @@ -26,12 +26,6 @@ spring.data.mongodb.password= spring.data.mongodb.authentication-database= mongodb.read-preference=secondaryPreferred -spring.datasource.driver-class-name=org.postgresql.Driver -spring.datasource.url= -spring.datasource.username= -spring.datasource.password= -spring.datasource.tomcat.max-active=3 - # Only to set up the database! # spring.jpa.generate-ddl=true diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/batch/io/AccessionWriterTest.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/batch/io/AccessionWriterTest.java index ab278fe38..d281e9e7f 100644 --- a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/batch/io/AccessionWriterTest.java +++ b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/batch/io/AccessionWriterTest.java @@ -25,15 +25,17 @@ import org.springframework.batch.item.Chunk; import org.springframework.batch.item.ExecutionContext; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.core.io.ClassPathResource; import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.jdbc.datasource.init.ResourceDatabasePopulator; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.ampt2d.commons.accession.core.models.AccessionWrapper; import uk.ac.ebi.eva.accession.core.batch.io.FastaSequenceReader; +import uk.ac.ebi.eva.accession.core.configuration.ContiguousIdBlocksDataSourceConfiguration; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.SubmittedVariantAccessioningConfiguration; import uk.ac.ebi.eva.accession.core.contig.ContigMapping; import uk.ac.ebi.eva.accession.core.contig.ContigNaming; @@ -43,17 +45,19 @@ import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; import uk.ac.ebi.eva.accession.core.service.GetOrCreateAccessionWrapperCreator; import uk.ac.ebi.eva.accession.core.service.nonhuman.SubmittedVariantAccessioningService; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.core.utils.PipelineTemporaryFolderUtil; import uk.ac.ebi.eva.accession.pipeline.batch.processors.VariantConverter; import uk.ac.ebi.eva.accession.pipeline.configuration.InputParametersConfiguration; import uk.ac.ebi.eva.accession.pipeline.configuration.batch.listeners.ListenersConfiguration; import uk.ac.ebi.eva.accession.pipeline.metric.AccessioningMetric; -import uk.ac.ebi.eva.accession.pipeline.test.MongoTestConfiguration; +import uk.ac.ebi.eva.accession.pipeline.test.BatchTestConfiguration; import uk.ac.ebi.eva.commons.core.models.pipeline.Variant; import uk.ac.ebi.eva.commons.core.models.pipeline.VariantSourceEntry; import uk.ac.ebi.eva.metrics.metric.MetricCompute; +import javax.sql.DataSource; import java.io.BufferedReader; import java.io.File; import java.io.FileInputStream; @@ -61,6 +65,8 @@ import java.io.InputStreamReader; import java.nio.file.Path; import java.nio.file.Paths; +import java.sql.Connection; +import java.sql.Statement; import java.time.LocalDateTime; import java.util.ArrayList; import java.util.Arrays; @@ -77,9 +83,9 @@ import static uk.ac.ebi.eva.accession.pipeline.batch.processors.ContigToGenbankReplacerProcessor.ORIGINAL_CHROMOSOME; @ExtendWith(SpringExtension.class) -@EnableAutoConfiguration @ContextConfiguration(classes = {SubmittedVariantAccessioningConfiguration.class, ListenersConfiguration.class, - InputParametersConfiguration.class, MongoTestConfiguration.class}) + InputParametersConfiguration.class, MongoTestConfiguration.class, ContiguousIdBlocksDataSourceConfiguration.class, +BatchTestConfiguration.class}) @TestPropertySource("classpath:accession-pipeline-test.properties") public class AccessionWriterTest extends MongoTestContainerHelper { @@ -136,6 +142,9 @@ public class AccessionWriterTest extends MongoTestContainerHelper { @Autowired private MetricCompute metricCompute; + @Autowired + private DataSource dataSource; + private ContigMapping contigMapping; @MockBean @@ -153,6 +162,14 @@ public class AccessionWriterTest extends MongoTestContainerHelper { @BeforeEach public void setUp() throws Exception { + try (Connection conn = dataSource.getConnection(); + Statement stmt = conn.createStatement()) { + stmt.execute("DROP TABLE IF EXISTS contiguous_id_blocks"); + } + ResourceDatabasePopulator populator = new ResourceDatabasePopulator(); + populator.addScript(new ClassPathResource("test-data/contiguous_id_blocks_schema.sql")); + populator.execute(dataSource); + contigMapping = new ContigMapping(Arrays.asList( new ContigSynonyms(CHROMOSOME_1, "assembled-molecule", "1", CONTIG_1, "refseq_1", "chr1", true), new ContigSynonyms(CHROMOSOME_2, "assembled-molecule", "2", CONTIG_2, "refseq_2", "chr2", true))); diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSeveralIntervalsOfSubsnpAccessionsJobConfigurationTest.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSeveralIntervalsOfSubsnpAccessionsJobConfigurationTest.java index b65a1f30f..3c7bb12a1 100644 --- a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSeveralIntervalsOfSubsnpAccessionsJobConfigurationTest.java +++ b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSeveralIntervalsOfSubsnpAccessionsJobConfigurationTest.java @@ -26,9 +26,11 @@ import org.springframework.batch.test.JobLauncherTestUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.core.io.ClassPathResource; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.http.HttpMethod; import org.springframework.http.HttpStatus; +import org.springframework.jdbc.datasource.init.ResourceDatabasePopulator; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; @@ -36,18 +38,19 @@ import org.springframework.test.web.client.ExpectedCount; import org.springframework.test.web.client.MockRestServiceServer; import org.springframework.web.client.RestTemplate; +import uk.ac.ebi.eva.accession.core.configuration.ContiguousIdBlocksDataSourceConfiguration; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.SubmittedVariantAccessioningConfiguration; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; import uk.ac.ebi.eva.accession.core.repository.nonhuman.eva.SubmittedVariantAccessioningRepository; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.pipeline.batch.io.AccessionReportWriter; import uk.ac.ebi.eva.accession.pipeline.parameters.InputParameters; -import uk.ac.ebi.eva.accession.pipeline.test.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.accession.pipeline.test.BatchTestConfiguration; -import uk.ac.ebi.eva.accession.pipeline.test.MongoTestConfiguration; import uk.ac.ebi.eva.commons.core.utils.FileUtils; import uk.ac.ebi.eva.metrics.count.CountServiceParameters; +import javax.sql.DataSource; import java.io.BufferedReader; import java.io.FileInputStream; import java.io.IOException; @@ -55,6 +58,8 @@ import java.net.URI; import java.nio.file.Files; import java.nio.file.Paths; +import java.sql.Connection; +import java.sql.Statement; import java.util.Arrays; import java.util.Collection; import java.util.Iterator; @@ -72,7 +77,7 @@ @ExtendWith(SpringExtension.class) @ContextConfiguration(classes = {BatchTestConfiguration.class, SubmittedVariantAccessioningConfiguration.class, - MongoTestConfiguration.class, BatchJobRepositoryTestConfiguration.class}) + MongoTestConfiguration.class, ContiguousIdBlocksDataSourceConfiguration.class}) @TestPropertySource("classpath:accession-pipeline-interval-test.properties") public class CreateSeveralIntervalsOfSubsnpAccessionsJobConfigurationTest extends MongoTestContainerHelper { @@ -94,6 +99,9 @@ public class CreateSeveralIntervalsOfSubsnpAccessionsJobConfigurationTest extend @Autowired private CountServiceParameters countServiceParameters; + @Autowired + private DataSource dataSource; + private final String URL_PATH_SAVE_COUNT = "/v1/bulk/count"; @Autowired @@ -110,6 +118,14 @@ public void setUp() throws Exception { mockServer.expect(ExpectedCount.manyTimes(), requestTo(new URI(countServiceParameters.getUrl() + URL_PATH_SAVE_COUNT))) .andExpect(method(HttpMethod.POST)) .andRespond(withStatus(HttpStatus.OK)); + + try (Connection conn = dataSource.getConnection(); + Statement stmt = conn.createStatement()) { + stmt.execute("DROP TABLE IF EXISTS contiguous_id_blocks"); + } + ResourceDatabasePopulator populator = new ResourceDatabasePopulator(); + populator.addScript(new ClassPathResource("test-data/contiguous_id_blocks_schema.sql")); + populator.execute(dataSource); } @AfterEach diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSubsnpAccessionsJobConfigurationTest.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSubsnpAccessionsJobConfigurationTest.java index 1673ca54a..d5b138d85 100644 --- a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSubsnpAccessionsJobConfigurationTest.java +++ b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSubsnpAccessionsJobConfigurationTest.java @@ -26,9 +26,11 @@ import org.springframework.batch.test.JobLauncherTestUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.core.io.ClassPathResource; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.http.HttpMethod; import org.springframework.http.HttpStatus; +import org.springframework.jdbc.datasource.init.ResourceDatabasePopulator; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; @@ -36,18 +38,19 @@ import org.springframework.test.web.client.ExpectedCount; import org.springframework.test.web.client.MockRestServiceServer; import org.springframework.web.client.RestTemplate; +import uk.ac.ebi.eva.accession.core.configuration.ContiguousIdBlocksDataSourceConfiguration; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.SubmittedVariantAccessioningConfiguration; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; import uk.ac.ebi.eva.accession.core.repository.nonhuman.eva.SubmittedVariantAccessioningRepository; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.pipeline.batch.io.AccessionReportWriter; import uk.ac.ebi.eva.accession.pipeline.parameters.InputParameters; -import uk.ac.ebi.eva.accession.pipeline.test.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.accession.pipeline.test.BatchTestConfiguration; -import uk.ac.ebi.eva.accession.pipeline.test.MongoTestConfiguration; import uk.ac.ebi.eva.commons.core.utils.FileUtils; import uk.ac.ebi.eva.metrics.count.CountServiceParameters; +import javax.sql.DataSource; import java.io.BufferedReader; import java.io.File; import java.io.FileInputStream; @@ -56,6 +59,8 @@ import java.net.URI; import java.nio.file.Files; import java.nio.file.Paths; +import java.sql.Connection; +import java.sql.Statement; import java.util.Collection; import java.util.Iterator; @@ -71,7 +76,7 @@ @ExtendWith(SpringExtension.class) @ContextConfiguration(classes = {BatchTestConfiguration.class, SubmittedVariantAccessioningConfiguration.class, - MongoTestConfiguration.class, BatchJobRepositoryTestConfiguration.class}) + MongoTestConfiguration.class, ContiguousIdBlocksDataSourceConfiguration.class}) @TestPropertySource("classpath:accession-pipeline-test.properties") public class CreateSubsnpAccessionsJobConfigurationTest extends MongoTestContainerHelper { @@ -99,6 +104,9 @@ public class CreateSubsnpAccessionsJobConfigurationTest extends MongoTestContain @Qualifier("COUNT_STATS_REST_TEMPLATE") private RestTemplate restTemplate; + @Autowired + private DataSource dataSource; + private MockRestServiceServer mockServer; @BeforeEach @@ -109,6 +117,14 @@ public void setUp() throws Exception { mockServer.expect(ExpectedCount.manyTimes(), requestTo(new URI(countServiceParameters.getUrl() + URL_PATH_SAVE_COUNT))) .andExpect(method(HttpMethod.POST)) .andRespond(withStatus(HttpStatus.OK)); + + try (Connection conn = dataSource.getConnection(); + Statement stmt = conn.createStatement()) { + stmt.execute("DROP TABLE IF EXISTS contiguous_id_blocks"); + } + ResourceDatabasePopulator populator = new ResourceDatabasePopulator(); + populator.addScript(new ClassPathResource("test-data/contiguous_id_blocks_schema.sql")); + populator.execute(dataSource); } @AfterEach diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSubsnpAccessionsRecoverStateTest.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSubsnpAccessionsRecoverStateTest.java index f1e8b3ad1..200067d5b 100644 --- a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSubsnpAccessionsRecoverStateTest.java +++ b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSubsnpAccessionsRecoverStateTest.java @@ -26,9 +26,11 @@ import org.springframework.batch.test.JobLauncherTestUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.core.io.ClassPathResource; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.http.HttpMethod; import org.springframework.http.HttpStatus; +import org.springframework.jdbc.datasource.init.ResourceDatabasePopulator; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit.jupiter.SpringExtension; @@ -37,24 +39,27 @@ import org.springframework.web.client.RestTemplate; import uk.ac.ebi.ampt2d.commons.accession.persistence.jpa.monotonic.entities.ContiguousIdBlock; import uk.ac.ebi.ampt2d.commons.accession.persistence.jpa.monotonic.repositories.ContiguousIdBlockRepository; +import uk.ac.ebi.eva.accession.core.configuration.ContiguousIdBlocksDataSourceConfiguration; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.SubmittedVariantAccessioningConfiguration; import uk.ac.ebi.eva.accession.core.model.SubmittedVariant; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; import uk.ac.ebi.eva.accession.core.repository.nonhuman.eva.SubmittedVariantAccessioningRepository; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.pipeline.batch.io.AccessionReportWriter; import uk.ac.ebi.eva.accession.pipeline.parameters.InputParameters; -import uk.ac.ebi.eva.accession.pipeline.test.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.accession.pipeline.test.BatchTestConfiguration; -import uk.ac.ebi.eva.accession.pipeline.test.MongoTestConfiguration; import uk.ac.ebi.eva.commons.core.utils.FileUtils; import uk.ac.ebi.eva.metrics.count.CountServiceParameters; +import javax.sql.DataSource; import java.io.FileInputStream; import java.io.IOException; import java.net.URI; import java.nio.file.Files; import java.nio.file.Paths; +import java.sql.Connection; +import java.sql.Statement; import java.util.ArrayList; import java.util.Collection; import java.util.Iterator; @@ -72,7 +77,7 @@ @ExtendWith(SpringExtension.class) @ContextConfiguration(classes = {BatchTestConfiguration.class, SubmittedVariantAccessioningConfiguration.class, - MongoTestConfiguration.class, BatchJobRepositoryTestConfiguration.class}) + MongoTestConfiguration.class, ContiguousIdBlocksDataSourceConfiguration.class}) @TestPropertySource("classpath:accession-pipeline-recover-state-test.properties") public class CreateSubsnpAccessionsRecoverStateTest extends MongoTestContainerHelper { @@ -96,6 +101,9 @@ public class CreateSubsnpAccessionsRecoverStateTest extends MongoTestContainerHe @Qualifier("COUNT_STATS_REST_TEMPLATE") private RestTemplate restTemplate; + @Autowired + private DataSource dataSource; + private final String URL_PATH_SAVE_COUNT = "/v1/bulk/count"; @Autowired @@ -113,6 +121,15 @@ public void setUp() throws Exception { mockServer.expect(ExpectedCount.manyTimes(), requestTo(new URI(countServiceParameters.getUrl() + URL_PATH_SAVE_COUNT))) .andExpect(method(HttpMethod.POST)) .andRespond(withStatus(HttpStatus.OK)); + + try (Connection conn = dataSource.getConnection(); + Statement stmt = conn.createStatement()) { + stmt.execute("DROP TABLE IF EXISTS contiguous_id_blocks"); + } + ResourceDatabasePopulator populator = new ResourceDatabasePopulator(); + populator.addScript(new ClassPathResource("test-data/contiguous_id_blocks_schema.sql")); + populator.addScript(new ClassPathResource("test-data/contiguous_id_blocks_recover_state_data.sql")); + populator.execute(dataSource); } @AfterEach @@ -207,7 +224,7 @@ private void verifyEndDBState() { assertEquals(5000000000l, block1.getFirstValue()); assertEquals(5000000029l, block1.getLastCommitted()); assertEquals(5000000029l, block1.getLastValue()); - assertEquals("0", block1.getApplicationInstanceId()); + assertEquals("1", block1.getApplicationInstanceId()); assertTrue(block1.isNotReserved()); // Block Recovered - (used the 5 unused accessions 5000000035 to 5000000039 and recovered others) @@ -215,7 +232,7 @@ private void verifyEndDBState() { assertEquals(5000000030l, block2.getFirstValue()); assertEquals(5000000059l, block2.getLastCommitted()); assertEquals(5000000059l, block2.getLastValue()); - assertEquals("0", block2.getApplicationInstanceId()); + assertEquals("1", block2.getApplicationInstanceId()); assertTrue(block2.isNotReserved()); // Block Recovered - (No accession used from this block as entire block was already used) @@ -223,7 +240,7 @@ private void verifyEndDBState() { assertEquals(5000000060l, block3.getFirstValue()); assertEquals(5000000089l, block3.getLastCommitted()); assertEquals(5000000089l, block3.getLastValue()); - assertEquals("0", block3.getApplicationInstanceId()); + assertEquals("1", block3.getApplicationInstanceId()); assertTrue(block3.isNotReserved()); // used the remaining 17 (22 - 5 (2nd block)) from 4th block @@ -231,7 +248,7 @@ private void verifyEndDBState() { assertEquals(5000000090l, block4.getFirstValue()); assertEquals(5000000106l, block4.getLastCommitted()); assertEquals(5000000119l, block4.getLastValue()); - assertEquals("0", block4.getApplicationInstanceId()); + assertEquals("1", block4.getApplicationInstanceId()); assertTrue(block4.isNotReserved()); } diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSubsnpAccessionsRecoveringStateJobConfigurationTest.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSubsnpAccessionsRecoveringStateJobConfigurationTest.java index 78ea91fb3..491bf40d6 100644 --- a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSubsnpAccessionsRecoveringStateJobConfigurationTest.java +++ b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSubsnpAccessionsRecoveringStateJobConfigurationTest.java @@ -26,9 +26,11 @@ import org.springframework.batch.test.JobLauncherTestUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.core.io.ClassPathResource; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.http.HttpMethod; import org.springframework.http.HttpStatus; +import org.springframework.jdbc.datasource.init.ResourceDatabasePopulator; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit.jupiter.SpringExtension; @@ -37,24 +39,27 @@ import org.springframework.web.client.RestTemplate; import uk.ac.ebi.ampt2d.commons.accession.persistence.jpa.monotonic.entities.ContiguousIdBlock; import uk.ac.ebi.ampt2d.commons.accession.persistence.jpa.monotonic.repositories.ContiguousIdBlockRepository; +import uk.ac.ebi.eva.accession.core.configuration.ContiguousIdBlocksDataSourceConfiguration; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.SubmittedVariantAccessioningConfiguration; import uk.ac.ebi.eva.accession.core.model.SubmittedVariant; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; import uk.ac.ebi.eva.accession.core.repository.nonhuman.eva.SubmittedVariantAccessioningRepository; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.pipeline.batch.io.AccessionReportWriter; import uk.ac.ebi.eva.accession.pipeline.parameters.InputParameters; -import uk.ac.ebi.eva.accession.pipeline.test.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.accession.pipeline.test.BatchTestConfiguration; -import uk.ac.ebi.eva.accession.pipeline.test.MongoTestConfiguration; import uk.ac.ebi.eva.commons.core.utils.FileUtils; import uk.ac.ebi.eva.metrics.count.CountServiceParameters; +import javax.sql.DataSource; import java.io.FileInputStream; import java.io.IOException; import java.net.URI; import java.nio.file.Files; import java.nio.file.Paths; +import java.sql.Connection; +import java.sql.Statement; import java.util.Collection; import java.util.Iterator; @@ -69,7 +74,7 @@ @ExtendWith(SpringExtension.class) @ContextConfiguration(classes = {BatchTestConfiguration.class, SubmittedVariantAccessioningConfiguration.class, - MongoTestConfiguration.class, BatchJobRepositoryTestConfiguration.class}) + MongoTestConfiguration.class, ContiguousIdBlocksDataSourceConfiguration.class}) @TestPropertySource("classpath:accession-pipeline-recover-test.properties") public class CreateSubsnpAccessionsRecoveringStateJobConfigurationTest extends MongoTestContainerHelper { @@ -100,6 +105,9 @@ public class CreateSubsnpAccessionsRecoveringStateJobConfigurationTest extends M @Autowired private MongoTemplate mongoTemplate; + @Autowired + private DataSource dataSource; + @Autowired @Qualifier(JOB_LAUNCHER_SUBSNP_ACCESSION_JOB) private JobLauncherTestUtils jobLauncherTestUtils; @@ -112,6 +120,16 @@ public void setUp() throws Exception { mockServer.expect(ExpectedCount.manyTimes(), requestTo(new URI(countServiceParameters.getUrl() + URL_PATH_SAVE_COUNT))) .andExpect(method(HttpMethod.POST)) .andRespond(withStatus(HttpStatus.OK)); + + try (Connection conn = dataSource.getConnection(); + Statement stmt = conn.createStatement()) { + stmt.execute("DROP TABLE IF EXISTS contiguous_id_blocks"); + } + + ResourceDatabasePopulator populator = new ResourceDatabasePopulator(); + populator.addScript(new ClassPathResource("test-data/contiguous_id_blocks_schema.sql")); + populator.addScript(new ClassPathResource("test-data/contiguous_id_blocks_data.sql")); + populator.execute(dataSource); } @AfterEach diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/DuplicateSSAccQCJobConfigurationTest.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/DuplicateSSAccQCJobConfigurationTest.java index 8d40527a4..a007e3a30 100644 --- a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/DuplicateSSAccQCJobConfigurationTest.java +++ b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/DuplicateSSAccQCJobConfigurationTest.java @@ -28,12 +28,12 @@ import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit.jupiter.SpringExtension; +import uk.ac.ebi.eva.accession.core.configuration.ContiguousIdBlocksDataSourceConfiguration; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpSubmittedVariantEntity; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; -import uk.ac.ebi.eva.accession.pipeline.test.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.accession.pipeline.test.BatchTestConfiguration; -import uk.ac.ebi.eva.accession.pipeline.test.MongoTestConfiguration; import java.io.IOException; import java.nio.file.Files; @@ -49,7 +49,7 @@ @ExtendWith(SpringExtension.class) @ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class, - BatchJobRepositoryTestConfiguration.class}) + ContiguousIdBlocksDataSourceConfiguration.class}) @TestPropertySource("classpath:duplicate-ss-acc-qc-test.properties") public class DuplicateSSAccQCJobConfigurationTest extends MongoTestContainerHelper { private static final String duplicateSsAccFile = "src/test/resources/duplicateSSAcc.csv"; diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/QCSubsnpAccessionsJobConfigurationTest.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/QCSubsnpAccessionsJobConfigurationTest.java index 9bf30a7ba..5a5de95cd 100644 --- a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/QCSubsnpAccessionsJobConfigurationTest.java +++ b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/QCSubsnpAccessionsJobConfigurationTest.java @@ -29,11 +29,11 @@ import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit.jupiter.SpringExtension; +import uk.ac.ebi.eva.accession.core.configuration.ContiguousIdBlocksDataSourceConfiguration; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.SubmittedVariantAccessioningConfiguration; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; -import uk.ac.ebi.eva.accession.pipeline.test.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.accession.pipeline.test.BatchTestConfiguration; -import uk.ac.ebi.eva.accession.pipeline.test.MongoTestConfiguration; import java.util.Collection; @@ -43,7 +43,7 @@ @ExtendWith(SpringExtension.class) @ContextConfiguration(classes = {BatchTestConfiguration.class, SubmittedVariantAccessioningConfiguration.class, - MongoTestConfiguration.class, BatchJobRepositoryTestConfiguration.class}) + MongoTestConfiguration.class, ContiguousIdBlocksDataSourceConfiguration.class}) @TestPropertySource("classpath:qc-accession-pipeline-test.properties") public class QCSubsnpAccessionsJobConfigurationTest extends MongoTestContainerHelper { @Autowired diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/CreateSubsnpAccessionsStepConfigurationTest.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/CreateSubsnpAccessionsStepConfigurationTest.java index 267a39dba..9dfe92d7d 100644 --- a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/CreateSubsnpAccessionsStepConfigurationTest.java +++ b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/CreateSubsnpAccessionsStepConfigurationTest.java @@ -29,28 +29,33 @@ import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.boot.test.mock.mockito.SpyBean; +import org.springframework.core.io.ClassPathResource; import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.jdbc.datasource.init.ResourceDatabasePopulator; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.ampt2d.commons.accession.persistence.jpa.monotonic.service.ContiguousIdBlockService; +import uk.ac.ebi.eva.accession.core.configuration.ContiguousIdBlocksDataSourceConfiguration; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.SubmittedVariantAccessioningConfiguration; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; import uk.ac.ebi.eva.accession.core.repository.nonhuman.eva.SubmittedVariantAccessioningRepository; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.pipeline.batch.io.AccessionReportWriter; import uk.ac.ebi.eva.accession.pipeline.batch.io.AccessionWriter; import uk.ac.ebi.eva.accession.pipeline.parameters.InputParameters; -import uk.ac.ebi.eva.accession.pipeline.test.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.accession.pipeline.test.BatchTestConfiguration; -import uk.ac.ebi.eva.accession.pipeline.test.MongoTestConfiguration; import uk.ac.ebi.eva.commons.core.utils.FileUtils; +import javax.sql.DataSource; import java.io.FileInputStream; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Paths; +import java.sql.Connection; +import java.sql.Statement; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.Mockito.when; @@ -59,7 +64,7 @@ @ExtendWith(SpringExtension.class) @ContextConfiguration(classes = {BatchTestConfiguration.class, SubmittedVariantAccessioningConfiguration.class, - MongoTestConfiguration.class, BatchJobRepositoryTestConfiguration.class}) + MongoTestConfiguration.class, ContiguousIdBlocksDataSourceConfiguration.class}) @TestPropertySource("classpath:accession-pipeline-test.properties") public class CreateSubsnpAccessionsStepConfigurationTest extends MongoTestContainerHelper { @@ -83,6 +88,9 @@ public class CreateSubsnpAccessionsStepConfigurationTest extends MongoTestContai @Autowired private InputParameters inputParameters; + @Autowired + private DataSource dataSource; + @Autowired private MongoTemplate mongoTemplate; @@ -91,6 +99,14 @@ public class CreateSubsnpAccessionsStepConfigurationTest extends MongoTestContai @BeforeEach public void setUp() throws Exception { + try (Connection conn = dataSource.getConnection(); + Statement stmt = conn.createStatement()) { + stmt.execute("DROP TABLE IF EXISTS contiguous_id_blocks"); + } + ResourceDatabasePopulator populator = new ResourceDatabasePopulator(); + populator.addScript(new ClassPathResource("test-data/contiguous_id_blocks_schema.sql")); + populator.execute(dataSource); + // if a new transaction is not created it will fail when(contiguousIdBlockService.getBlockParameters("test-pipeline-ss")) .thenThrow(RuntimeException.class) diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/QCSubsnpAccessionsStepConfigurationTest.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/QCSubsnpAccessionsStepConfigurationTest.java index 1737d3d62..ac14e6f9a 100644 --- a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/QCSubsnpAccessionsStepConfigurationTest.java +++ b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/QCSubsnpAccessionsStepConfigurationTest.java @@ -31,12 +31,12 @@ import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit.jupiter.SpringExtension; +import uk.ac.ebi.eva.accession.core.configuration.ContiguousIdBlocksDataSourceConfiguration; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.SubmittedVariantAccessioningConfiguration; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.pipeline.batch.io.AccessionWriter; -import uk.ac.ebi.eva.accession.pipeline.test.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.accession.pipeline.test.BatchTestConfiguration; -import uk.ac.ebi.eva.accession.pipeline.test.MongoTestConfiguration; import static org.junit.jupiter.api.Assertions.assertEquals; import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.QC_SUBSNP_ACCESSION_STEP; @@ -44,7 +44,7 @@ @ExtendWith(SpringExtension.class) @ContextConfiguration(classes = {BatchTestConfiguration.class, SubmittedVariantAccessioningConfiguration.class, - MongoTestConfiguration.class, BatchJobRepositoryTestConfiguration.class}) + MongoTestConfiguration.class, ContiguousIdBlocksDataSourceConfiguration.class}) @TestPropertySource("classpath:qc-accession-pipeline-test.properties") public class QCSubsnpAccessionsStepConfigurationTest extends MongoTestContainerHelper { @Autowired diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/runner/EvaAccessionJobLauncherCommandLineRunnerTest.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/runner/EvaAccessionJobLauncherCommandLineRunnerTest.java index 660cef9cb..2f8d20c8d 100644 --- a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/runner/EvaAccessionJobLauncherCommandLineRunnerTest.java +++ b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/runner/EvaAccessionJobLauncherCommandLineRunnerTest.java @@ -20,63 +20,54 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; -import org.springframework.batch.core.JobInstance; -import org.springframework.batch.core.StepExecution; import org.springframework.batch.core.explore.JobExplorer; import org.springframework.batch.core.repository.JobRepository; import org.springframework.batch.test.JobRepositoryTestUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.core.io.ClassPathResource; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.http.HttpMethod; import org.springframework.http.HttpStatus; -import org.springframework.test.annotation.DirtiesContext; +import org.springframework.jdbc.datasource.init.ResourceDatabasePopulator; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit.jupiter.SpringExtension; import org.springframework.test.web.client.ExpectedCount; import org.springframework.test.web.client.MockRestServiceServer; import org.springframework.web.client.RestTemplate; +import uk.ac.ebi.eva.accession.core.configuration.ContiguousIdBlocksDataSourceConfiguration; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; -import uk.ac.ebi.eva.accession.core.runner.CommandLineRunnerUtils; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.pipeline.parameters.InputParameters; -import uk.ac.ebi.eva.accession.pipeline.test.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.accession.pipeline.test.BatchTestConfiguration; -import uk.ac.ebi.eva.accession.pipeline.test.MongoTestConfiguration; import uk.ac.ebi.eva.commons.batch.io.VcfReader; import uk.ac.ebi.eva.metrics.count.CountServiceParameters; +import javax.sql.DataSource; import java.io.File; import java.net.URI; import java.nio.file.Files; -import java.nio.file.NoSuchFileException; import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.StandardCopyOption; -import java.util.Arrays; -import java.util.Collections; +import java.sql.Connection; +import java.sql.Statement; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.springframework.test.web.client.match.MockRestRequestMatchers.method; import static org.springframework.test.web.client.match.MockRestRequestMatchers.requestTo; import static org.springframework.test.web.client.response.MockRestResponseCreators.withStatus; import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.SUBSNP_ACCESSION_JOB; -import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.SUBSNP_ACCESSION_STEP; import static uk.ac.ebi.eva.accession.pipeline.runner.RunnerUtil.deleteTemporaryContigAndVariantFiles; import static uk.ac.ebi.eva.accession.pipeline.runner.RunnerUtil.getOriginalVcfContent; -import static uk.ac.ebi.eva.accession.pipeline.runner.RunnerUtil.injectErrorIntoTempVcf; -import static uk.ac.ebi.eva.accession.pipeline.runner.RunnerUtil.remediateTempVcfError; import static uk.ac.ebi.eva.accession.pipeline.runner.RunnerUtil.useOriginalVcfFile; -import static uk.ac.ebi.eva.accession.pipeline.runner.RunnerUtil.useTempVcfFile; import static uk.ac.ebi.eva.accession.pipeline.runner.RunnerUtil.writeToTempVCFFile; @ExtendWith(SpringExtension.class) @ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class, - BatchJobRepositoryTestConfiguration.class}) + ContiguousIdBlocksDataSourceConfiguration.class}) @TestPropertySource("classpath:accession-pipeline-test.properties") public class EvaAccessionJobLauncherCommandLineRunnerTest extends MongoTestContainerHelper { @@ -117,12 +108,13 @@ public class EvaAccessionJobLauncherCommandLineRunnerTest extends MongoTestConta private static String originalVcfInputFilePath; - private static String originalVcfOutputFilePath; - private static String originalVcfContent; private boolean originalInputParametersCaptured = false; + @Autowired + private DataSource dataSource; + @BeforeAll public static void initializeTempFile() throws Exception { tempVcfInputFileToTestFailingJobs = File.createTempFile("resumeFailingJob", ".vcf.gz"); @@ -138,7 +130,6 @@ public static void deleteTempFile() { public void setUp() throws Exception { if (!originalInputParametersCaptured) { originalVcfInputFilePath = inputParameters.getVcf(); - originalVcfOutputFilePath = inputParameters.getOutputVcf(); originalVcfContent = getOriginalVcfContent(originalVcfInputFilePath); writeToTempVCFFile(originalVcfContent, tempVcfInputFileToTestFailingJobs); originalInputParametersCaptured = true; @@ -154,12 +145,19 @@ public void setUp() throws Exception { .andRespond(withStatus(HttpStatus.OK)); mongoTemplate.dropCollection(SubmittedVariantEntity.class); + + try (Connection conn = dataSource.getConnection(); + Statement stmt = conn.createStatement()) { + stmt.execute("DROP TABLE IF EXISTS contiguous_id_blocks"); + } + ResourceDatabasePopulator populator = new ResourceDatabasePopulator(); + populator.addScript(new ClassPathResource("test-data/contiguous_id_blocks_schema.sql")); + populator.execute(dataSource); } @AfterEach public void tearDown() { jobRepositoryTestUtils.removeJobExecutions(); - inputParameters.setForceRestart(false); } @Test @@ -176,129 +174,4 @@ public void runJobWithNoName() throws Exception { assertEquals(EvaAccessionJobLauncherCommandLineRunner.EXIT_WITH_ERRORS, runner.getExitCode()); } - - @Test - @DirtiesContext - @Disabled - public void restartCompletedJobThatIsAlreadyInTheRepository() throws Exception { - runner.run(); - assertEquals(EvaAccessionJobLauncherCommandLineRunner.EXIT_WITHOUT_ERRORS, runner.getExitCode()); - - deleteTemporaryContigAndVariantFiles(inputParameters, tempVcfOutputDir); - - inputParameters.setForceRestart(true); - runner.run(); - assertEquals(EvaAccessionJobLauncherCommandLineRunner.EXIT_WITHOUT_ERRORS, runner.getExitCode()); - } - - - private JobInstance runJobAandCheckResults() throws Exception { - runner.run(); - assertEquals(EvaAccessionJobLauncherCommandLineRunner.EXIT_WITH_ERRORS, runner.getExitCode()); - JobInstance currentJobInstance = CommandLineRunnerUtils.getLastJobExecution(SUBSNP_ACCESSION_JOB, - jobExplorer, - inputParameters.toJobParameters()) - .getJobInstance(); - StepExecution stepExecution = jobRepository.getLastStepExecution(currentJobInstance, SUBSNP_ACCESSION_STEP); - //Ensure that only the first batch was written (batch size is 5 and error was at line#9) - assertEquals(inputParameters.getChunkSize(), stepExecution.getWriteCount()); - - return currentJobInstance; - } - - @Test - @DirtiesContext - public void forceRestartButNoJobInTheRepository() throws Exception { - inputParameters.setForceRestart(true); - assertEquals(Collections.EMPTY_LIST, jobExplorer.getJobNames()); - runner.run(); - - assertEquals(EvaAccessionJobLauncherCommandLineRunner.EXIT_WITHOUT_ERRORS, runner.getExitCode()); - } - - @Test - @DirtiesContext - @Disabled - public void resumeFailingJobFromCorrectChunk() throws Exception { - // Jobs A, B, C are run chronological order; A and C have SAME parameters; - // A is the job that is run after VCF fault injection (as part of the runTestWithFaultInjection method), - // therefore should fail. - // B is a job run with the original VCF without any faults (run separately), therefore should succeed. - // C is a job with the same parameters as A run after VCF fault remediation (as part of the - // runTestWithFaultInjection method), therefore should resume A and succeed. - - useTempVcfFile(inputParameters, tempVcfInputFileToTestFailingJobs, vcfReader); - String modifiedVcfContent = originalVcfContent.replace("76852", "76852jibberish"); - injectErrorIntoTempVcf(modifiedVcfContent, tempVcfInputFileToTestFailingJobs); - JobInstance failingJobInstance = runJobAandCheckResults(); - - runJobBAndCheckResults(); - - remediateTempVcfError(originalVcfContent, tempVcfInputFileToTestFailingJobs); - runJobCAndCheckResumption(failingJobInstance); - } - - private void runJobBAndCheckResults() throws Exception { - useOriginalVcfFile(inputParameters, originalVcfInputFilePath, vcfReader); - // Back up contig and variant files (left behind by previous unsuccessful job A) to temp folder - // so as to not interfere with this job's execution which uses the original VCF file - backUpContigAndVariantFilesToTempFolder(); - - runner.run(); - assertEquals(EvaAccessionJobLauncherCommandLineRunner.EXIT_WITHOUT_ERRORS, runner.getExitCode()); - - //Restore state so that Job C can continue running after fault remediation - useTempVcfFile(inputParameters, tempVcfInputFileToTestFailingJobs, vcfReader); - restoreContigAndVariantFilesFromTempFolder(); - } - - private void runJobCAndCheckResumption(JobInstance failingJobInstance) throws Exception { - runner.run(); - JobInstance currentJobInstance = CommandLineRunnerUtils.getLastJobExecution(SUBSNP_ACCESSION_JOB, - jobExplorer, - inputParameters.toJobParameters()) - .getJobInstance(); - StepExecution stepExecution = jobRepository.getLastStepExecution(currentJobInstance, SUBSNP_ACCESSION_STEP); - // Did we resume the previous failed job instance? - assertEquals(failingJobInstance.getInstanceId(), currentJobInstance.getInstanceId()); - - int numberOfLinesInVcf = getNumberOfLinesInVcfString(originalVcfContent); - int numberOfNonVariants = 1; //TBGI000010 is a non-variant - // Test resumption point - did we pick up where we left off? - // Ensure all the batches other than the first batch were processed - assertEquals(numberOfLinesInVcf - inputParameters.getChunkSize() - numberOfNonVariants, - stepExecution.getWriteCount()); - assertEquals(EvaAccessionJobLauncherCommandLineRunner.EXIT_WITHOUT_ERRORS, runner.getExitCode()); - } - - - private void backUpContigAndVariantFilesToTempFolder() { - moveFile(Paths.get(originalVcfOutputFilePath + ".contigs"), - Paths.get(tempVcfOutputDir + "/accession-output.vcf.contigs")); - moveFile(Paths.get(originalVcfOutputFilePath + ".variants"), - Paths.get(tempVcfOutputDir + "/accession-output.vcf.variants")); - } - - private void restoreContigAndVariantFilesFromTempFolder() { - moveFile(Paths.get(tempVcfOutputDir + "/accession-output.vcf.contigs"), - Paths.get(Paths.get(originalVcfOutputFilePath).getParent() + "/accession-output.vcf.contigs")); - moveFile(Paths.get(tempVcfOutputDir + "/accession-output.vcf.variants"), - Paths.get(Paths.get(originalVcfOutputFilePath).getParent() + "/accession-output.vcf.variants")); - } - - private void moveFile(Path source, Path destination) { - try { - Files.move(source, destination, StandardCopyOption.REPLACE_EXISTING); - } catch (Exception ex) { - if (!(ex instanceof NoSuchFileException)) { - throw new RuntimeException(ex); - } - } - } - - private int getNumberOfLinesInVcfString(String vcfString) { - return (int) Arrays.stream(vcfString.split(System.lineSeparator())) - .filter(line -> !line.startsWith("#")) - .count(); - } } \ No newline at end of file diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/runner/JobFailureBlocksReleasedTest.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/runner/JobFailureBlocksReleasedTest.java index cbbb5eebc..15742b21d 100644 --- a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/runner/JobFailureBlocksReleasedTest.java +++ b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/runner/JobFailureBlocksReleasedTest.java @@ -22,23 +22,28 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.io.ClassPathResource; import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.jdbc.datasource.init.ResourceDatabasePopulator; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.ampt2d.commons.accession.persistence.jpa.monotonic.repositories.ContiguousIdBlockRepository; +import uk.ac.ebi.eva.accession.core.configuration.ContiguousIdBlocksDataSourceConfiguration; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.pipeline.parameters.InputParameters; -import uk.ac.ebi.eva.accession.pipeline.test.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.accession.pipeline.test.BatchTestConfiguration; -import uk.ac.ebi.eva.accession.pipeline.test.MongoTestConfiguration; import uk.ac.ebi.eva.commons.batch.io.VcfReader; +import javax.sql.DataSource; import java.io.File; import java.nio.file.Files; import java.nio.file.Path; +import java.sql.Connection; +import java.sql.Statement; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -52,7 +57,7 @@ @ExtendWith(SpringExtension.class) @ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class, - BatchJobRepositoryTestConfiguration.class}) + ContiguousIdBlocksDataSourceConfiguration.class}) @TestPropertySource("classpath:accession-pipeline-test.properties") public class JobFailureBlocksReleasedTest extends MongoTestContainerHelper { @Autowired @@ -78,6 +83,9 @@ public class JobFailureBlocksReleasedTest extends MongoTestContainerHelper { @Autowired private ContiguousIdBlockRepository blockRepository; + @Autowired + private DataSource dataSource; + @BeforeAll public static void initializeTempFile() throws Exception { tempVcfInputFileToTestFailingJobs = File.createTempFile("resumeFailingJob", ".vcf.gz"); @@ -100,6 +108,15 @@ public void setUp() throws Exception { useOriginalVcfFile(inputParameters, originalVcfInputFilePath, vcfReader); mongoTemplate.dropCollection(SubmittedVariantEntity.class); + + try (Connection conn = dataSource.getConnection(); + Statement stmt = conn.createStatement()) { + stmt.execute("DROP TABLE IF EXISTS contiguous_id_blocks"); + } + ResourceDatabasePopulator populator = new ResourceDatabasePopulator(); + populator.addScript(new ClassPathResource("test-data/contiguous_id_blocks_schema.sql")); + populator.execute(dataSource); + } @Test diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/runner/RestartFailedJobTest.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/runner/RestartFailedJobTest.java deleted file mode 100644 index ea0cd06df..000000000 --- a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/runner/RestartFailedJobTest.java +++ /dev/null @@ -1,214 +0,0 @@ -/* - * Copyright 2014-2018 EMBL - European Bioinformatics Institute - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package uk.ac.ebi.eva.accession.pipeline.runner; - -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.springframework.batch.core.JobInstance; -import org.springframework.batch.core.StepExecution; -import org.springframework.batch.core.explore.JobExplorer; -import org.springframework.batch.core.repository.JobRepository; -import org.springframework.batch.test.JobRepositoryTestUtils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.boot.test.mock.mockito.SpyBean; -import org.springframework.data.mongodb.core.MongoTemplate; -import org.springframework.http.HttpMethod; -import org.springframework.http.HttpStatus; -import org.springframework.test.annotation.DirtiesContext; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit.jupiter.SpringExtension; -import org.springframework.test.web.client.ExpectedCount; -import org.springframework.test.web.client.MockRestServiceServer; -import org.springframework.web.client.RestTemplate; -import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; -import uk.ac.ebi.eva.accession.core.runner.CommandLineRunnerUtils; -import uk.ac.ebi.eva.accession.core.service.nonhuman.SubmittedVariantAccessioningService; -import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; -import uk.ac.ebi.eva.accession.pipeline.parameters.InputParameters; -import uk.ac.ebi.eva.accession.pipeline.test.BatchJobRepositoryTestConfiguration; -import uk.ac.ebi.eva.accession.pipeline.test.BatchTestConfiguration; -import uk.ac.ebi.eva.accession.pipeline.test.MongoTestConfiguration; -import uk.ac.ebi.eva.commons.batch.io.VcfReader; -import uk.ac.ebi.eva.metrics.count.CountServiceParameters; - -import java.io.File; -import java.net.URI; -import java.nio.file.Files; -import java.nio.file.Path; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotEquals; -import static org.mockito.Mockito.doNothing; -import static org.springframework.test.web.client.match.MockRestRequestMatchers.method; -import static org.springframework.test.web.client.match.MockRestRequestMatchers.requestTo; -import static org.springframework.test.web.client.response.MockRestResponseCreators.withStatus; -import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.SUBSNP_ACCESSION_JOB; -import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.SUBSNP_ACCESSION_STEP; -import static uk.ac.ebi.eva.accession.pipeline.runner.RunnerUtil.deleteTemporaryContigAndVariantFiles; -import static uk.ac.ebi.eva.accession.pipeline.runner.RunnerUtil.getOriginalVcfContent; -import static uk.ac.ebi.eva.accession.pipeline.runner.RunnerUtil.injectErrorIntoTempVcf; -import static uk.ac.ebi.eva.accession.pipeline.runner.RunnerUtil.remediateTempVcfError; -import static uk.ac.ebi.eva.accession.pipeline.runner.RunnerUtil.useOriginalVcfFile; -import static uk.ac.ebi.eva.accession.pipeline.runner.RunnerUtil.useTempVcfFile; -import static uk.ac.ebi.eva.accession.pipeline.runner.RunnerUtil.writeToTempVCFFile; - -@ExtendWith(SpringExtension.class) -@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class, - BatchJobRepositoryTestConfiguration.class}) -@TestPropertySource("classpath:accession-pipeline-test.properties") -public class RestartFailedJobTest extends MongoTestContainerHelper { - - @Autowired - private InputParameters inputParameters; - - @Autowired - private JobRepository jobRepository; - - @Autowired - private JobExplorer jobExplorer; - - @Autowired - private EvaAccessionJobLauncherCommandLineRunner runner; - - @Autowired - private VcfReader vcfReader; - - @Autowired - private CountServiceParameters countServiceParameters; - - @Autowired - @Qualifier("COUNT_STATS_REST_TEMPLATE") - private RestTemplate restTemplate; - - @Autowired - private MongoTemplate mongoTemplate; - - private final String URL_PATH_SAVE_COUNT = "/v1/bulk/count"; - - private MockRestServiceServer mockServer; - - private JobRepositoryTestUtils jobRepositoryTestUtils; - - private static File tempVcfInputFileToTestFailingJobs; - - private static Path tempVcfOutputDir; - - private static String originalVcfInputFilePath; - - private static String originalVcfContent; - - private boolean originalInputParametersCaptured = false; - - @SpyBean - private SubmittedVariantAccessioningService accessioningServiceSpy; - - @BeforeAll - public static void initializeTempFile() throws Exception { - tempVcfInputFileToTestFailingJobs = File.createTempFile("resumeFailingJob", ".vcf.gz"); - tempVcfOutputDir = Files.createTempDirectory("contigs_variants_dir"); - } - - @AfterAll - public static void deleteTempFile() { - tempVcfInputFileToTestFailingJobs.delete(); - } - - @BeforeEach - public void setUp() throws Exception { - if (!originalInputParametersCaptured) { - originalVcfInputFilePath = inputParameters.getVcf(); - originalVcfContent = getOriginalVcfContent(originalVcfInputFilePath); - writeToTempVCFFile(originalVcfContent, tempVcfInputFileToTestFailingJobs); - originalInputParametersCaptured = true; - } - jobRepositoryTestUtils = new JobRepositoryTestUtils(jobRepository); - runner.setJobName(SUBSNP_ACCESSION_JOB); - deleteTemporaryContigAndVariantFiles(inputParameters, tempVcfOutputDir); - useOriginalVcfFile(inputParameters, originalVcfInputFilePath, vcfReader); - - mockServer = MockRestServiceServer.createServer(restTemplate); - mockServer.expect(ExpectedCount.manyTimes(), requestTo(new URI(countServiceParameters.getUrl() + URL_PATH_SAVE_COUNT))) - .andExpect(method(HttpMethod.POST)) - .andRespond(withStatus(HttpStatus.OK)); - - mongoTemplate.dropCollection(SubmittedVariantEntity.class); - // Mock the behavior of shutDownAccessionGenerator method to do nothing - doNothing().when(accessioningServiceSpy).shutDownAccessionGenerator(); - - } - - @AfterEach - public void tearDown() { - jobRepositoryTestUtils.removeJobExecutions(); - inputParameters.setForceRestart(false); - } - - /* - * Separated this test from the rest of the tests in EvaAccessionJobLauncherCommandLineRunnerTest, - * as we have to Mock(Spy to be exact) on the SubmittedVariantAccessioningService bean in order to reuse the same - * without shutting down its accession generator. - * - * Ideally, we should not be Spying but rather the jobs when restarting should be using a new instance of the service. - * but it was tricky to inject, hence the workaround. - * */ - @Test - @DirtiesContext - public void restartFailedJobThatIsAlreadyInTheRepository() throws Exception { - useTempVcfFile(inputParameters, tempVcfInputFileToTestFailingJobs, vcfReader); - String modifiedVcfContent = originalVcfContent.replace("76852", "76852jibberish"); - injectErrorIntoTempVcf(modifiedVcfContent, tempVcfInputFileToTestFailingJobs); - JobInstance failingJobInstance = runJobAandCheckResults(); - - mongoTemplate.dropCollection(SubmittedVariantEntity.class); - - inputParameters.setForceRestart(true); - remediateTempVcfError(originalVcfContent, tempVcfInputFileToTestFailingJobs); - deleteTemporaryContigAndVariantFiles(inputParameters, tempVcfOutputDir); //left behind by unsuccessful job A - runJobBAndCheckRestart(failingJobInstance); - } - - private JobInstance runJobAandCheckResults() throws Exception { - runner.run(); - assertEquals(EvaAccessionJobLauncherCommandLineRunner.EXIT_WITH_ERRORS, runner.getExitCode()); - JobInstance currentJobInstance = CommandLineRunnerUtils.getLastJobExecution(SUBSNP_ACCESSION_JOB, - jobExplorer, - inputParameters.toJobParameters()) - .getJobInstance(); - StepExecution stepExecution = jobRepository.getLastStepExecution(currentJobInstance, SUBSNP_ACCESSION_STEP); - //Ensure that only the first batch was written (batch size is 5 and error was at line#9) - assertEquals(inputParameters.getChunkSize(), stepExecution.getWriteCount()); - - return currentJobInstance; - } - - private void runJobBAndCheckRestart(JobInstance failingJobInstance) throws Exception { - runner.run(); - assertEquals(EvaAccessionJobLauncherCommandLineRunner.EXIT_WITHOUT_ERRORS, runner.getExitCode()); - JobInstance currentJobInstance = CommandLineRunnerUtils.getLastJobExecution(SUBSNP_ACCESSION_JOB, - jobExplorer, - inputParameters.toJobParameters()) - .getJobInstance(); - assertNotEquals(failingJobInstance.getInstanceId(), currentJobInstance.getInstanceId()); - } - -} \ No newline at end of file diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/runner/SSAccessionRecoveryTest.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/runner/SSAccessionRecoveryTest.java index dacbf692c..0fbcdc038 100644 --- a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/runner/SSAccessionRecoveryTest.java +++ b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/runner/SSAccessionRecoveryTest.java @@ -16,24 +16,32 @@ * limitations under the License. */ +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.batch.test.context.SpringBatchTest; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.io.ClassPathResource; +import org.springframework.jdbc.datasource.init.ResourceDatabasePopulator; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.ampt2d.commons.accession.persistence.jpa.monotonic.entities.ContiguousIdBlock; import uk.ac.ebi.ampt2d.commons.accession.persistence.jpa.monotonic.repositories.ContiguousIdBlockRepository; +import uk.ac.ebi.eva.accession.core.configuration.ContiguousIdBlocksDataSourceConfiguration; +import uk.ac.ebi.eva.accession.core.configuration.InMemoryBatchConfiguration; import uk.ac.ebi.eva.accession.core.model.SubmittedVariant; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; import uk.ac.ebi.eva.accession.core.repository.nonhuman.eva.SubmittedVariantAccessioningRepository; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; -import uk.ac.ebi.eva.accession.pipeline.test.BatchJobRepositoryTestConfiguration; -import uk.ac.ebi.eva.accession.pipeline.test.MongoTestConfiguration; +import uk.ac.ebi.eva.accession.pipeline.test.BatchTestConfiguration; import uk.ac.ebi.eva.accession.pipeline.test.SSAccessionRecoveryTestConfiguration; +import javax.sql.DataSource; +import java.sql.Connection; +import java.sql.Statement; import java.util.ArrayList; import java.util.List; @@ -43,7 +51,7 @@ @ExtendWith(SpringExtension.class) @ContextConfiguration(classes = {SSAccessionRecoveryTestConfiguration.class, MongoTestConfiguration.class, - BatchJobRepositoryTestConfiguration.class}) + ContiguousIdBlocksDataSourceConfiguration.class, InMemoryBatchConfiguration.class, BatchTestConfiguration.class}) @TestPropertySource("classpath:ss-accession-recovery.properties") @SpringBatchTest public class SSAccessionRecoveryTest extends MongoTestContainerHelper { @@ -56,6 +64,22 @@ public class SSAccessionRecoveryTest extends MongoTestContainerHelper { @Autowired private SubmittedVariantAccessioningRepository mongoRepository; + @Autowired + private DataSource dataSource; + + @BeforeEach + public void setUp() throws Exception { + try (Connection conn = dataSource.getConnection(); + Statement stmt = conn.createStatement()) { + stmt.execute("DROP TABLE IF EXISTS contiguous_id_blocks"); + } + + ResourceDatabasePopulator populator = new ResourceDatabasePopulator(); + populator.addScript(new ClassPathResource("test-data/contiguous_id_blocks_schema.sql")); + populator.addScript(new ClassPathResource("test-data/ss_accession_recovery_test_data.sql")); + populator.execute(dataSource); + } + @Test @DirtiesContext public void testContiguousBlocksForCategorySSAreRecovered() throws Exception { @@ -192,7 +216,7 @@ private void verifyEndDBState() { assertEquals(5000000000l, block1.getFirstValue()); assertEquals(5000000029l, block1.getLastCommitted()); assertEquals(5000000029l, block1.getLastValue()); - assertEquals("0", block1.getApplicationInstanceId()); + assertEquals("1", block1.getApplicationInstanceId()); assertTrue(block1.isNotReserved()); // Block Recovered partially - (used 5000000030-5000000034 and 5000000040-5000000059) @@ -201,7 +225,7 @@ private void verifyEndDBState() { assertEquals(5000000030l, block2.getFirstValue()); assertEquals(5000000034l, block2.getLastCommitted()); assertEquals(5000000059l, block2.getLastValue()); - assertEquals("0", block2.getApplicationInstanceId()); + assertEquals("1", block2.getApplicationInstanceId()); assertTrue(block2.isNotReserved()); // Block Recovered - recovered entire block @@ -209,7 +233,7 @@ private void verifyEndDBState() { assertEquals(5000000060l, block3.getFirstValue()); assertEquals(5000000089l, block3.getLastCommitted()); assertEquals(5000000089l, block3.getLastValue()); - assertEquals("0", block3.getApplicationInstanceId()); + assertEquals("1", block3.getApplicationInstanceId()); assertTrue(block3.isNotReserved()); // Block Recovered - None of the accessions are used, just released the block @@ -217,7 +241,7 @@ private void verifyEndDBState() { assertEquals(5000000090l, block4.getFirstValue()); assertEquals(5000000089l, block4.getLastCommitted()); assertEquals(5000000119l, block4.getLastValue()); - assertEquals("0", block4.getApplicationInstanceId()); + assertEquals("1", block4.getApplicationInstanceId()); assertTrue(block4.isNotReserved()); // Block Not Recovered - Block not to be recovered as it's last update is after cut off time diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/test/BatchJobRepositoryTestConfiguration.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/test/BatchJobRepositoryTestConfiguration.java deleted file mode 100644 index 776d64a87..000000000 --- a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/test/BatchJobRepositoryTestConfiguration.java +++ /dev/null @@ -1,22 +0,0 @@ -package uk.ac.ebi.eva.accession.pipeline.test; - -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Primary; -import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseBuilder; -import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseType; - -import javax.sql.DataSource; - -@Configuration -public class BatchJobRepositoryTestConfiguration { - - @Bean - @Primary - public DataSource dataSource() { - return new EmbeddedDatabaseBuilder() - .setType(EmbeddedDatabaseType.HSQL) - .addScript("org/springframework/batch/core/schema-hsqldb.sql") - .build(); - } -} \ No newline at end of file diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/test/BatchTestConfiguration.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/test/BatchTestConfiguration.java index 38a6459e5..024862093 100644 --- a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/test/BatchTestConfiguration.java +++ b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/test/BatchTestConfiguration.java @@ -21,9 +21,10 @@ import org.springframework.batch.core.repository.JobRepository; import org.springframework.batch.test.JobLauncherTestUtils; import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Import; +import org.springframework.web.client.RestTemplate; +import uk.ac.ebi.eva.accession.core.configuration.InMemoryBatchConfiguration; import uk.ac.ebi.eva.accession.pipeline.configuration.batch.io.AccessionWriterConfiguration; import uk.ac.ebi.eva.accession.pipeline.configuration.batch.io.DuplicateSSAccQCWriterConfiguration; import uk.ac.ebi.eva.accession.pipeline.configuration.batch.io.SSAccFileReaderConfiguration; @@ -48,8 +49,7 @@ import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.QC_SUBSNP_ACCESSION_JOB; import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.SUBSNP_ACCESSION_JOB; -@EnableAutoConfiguration -@Import({SubsnpAccessionsJobConfiguration.class, SubsnpAccessionsStepConfiguration.class, +@Import({InMemoryBatchConfiguration.class, SubsnpAccessionsJobConfiguration.class, SubsnpAccessionsStepConfiguration.class, QCSubsnpAccessionsJobConfiguration.class, QCSubsnpAccessionsStepConfiguration.class, VcfReaderConfiguration.class, VariantProcessorConfiguration.class, AccessionWriterConfiguration.class, BuildReportStepConfiguration.class, AccessioningShutdownStepConfiguration.class, @@ -97,4 +97,9 @@ public JobLauncherTestUtils jobLauncherTestUtilsDuplicateSSAccQC(JobLauncher job public JobExecutionApplicationListener jobExecutionApplicationListener() { return new JobExecutionApplicationListener(); } + + @Bean(name = "COUNT_STATS_REST_TEMPLATE") + public RestTemplate countStatsRestTemplate() { + return new RestTemplate(); + } } \ No newline at end of file diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/test/MongoTestConfiguration.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/test/MongoTestConfiguration.java deleted file mode 100644 index 2a261f1b6..000000000 --- a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/test/MongoTestConfiguration.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Copyright 2021 EMBL - European Bioinformatics Institute - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package uk.ac.ebi.eva.accession.pipeline.test; - -import com.mongodb.ReadPreference; -import com.mongodb.WriteConcern; -import com.mongodb.client.MongoClient; -import com.mongodb.client.MongoClients; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.domain.EntityScan; -import org.springframework.boot.test.autoconfigure.data.mongo.AutoConfigureDataMongo; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Primary; -import org.springframework.data.mongodb.MongoDatabaseFactory; -import org.springframework.data.mongodb.config.EnableMongoAuditing; -import org.springframework.data.mongodb.core.MongoTemplate; -import org.springframework.data.mongodb.core.SimpleMongoClientDatabaseFactory; -import org.springframework.data.mongodb.core.WriteResultChecking; -import org.springframework.data.mongodb.core.convert.DefaultDbRefResolver; -import org.springframework.data.mongodb.core.convert.DefaultMongoTypeMapper; -import org.springframework.data.mongodb.core.convert.MappingMongoConverter; -import org.springframework.data.mongodb.core.convert.MongoCustomConversions; -import org.springframework.data.mongodb.core.mapping.MongoMappingContext; -import org.springframework.data.mongodb.repository.config.EnableMongoRepositories; - -import java.util.Collections; - -@Configuration -@EntityScan(basePackages = {"uk.ac.ebi.eva.accession.core.persistence"}) -@EnableMongoRepositories(basePackages = { - "uk.ac.ebi.eva.accession.core.persistence", - "uk.ac.ebi.ampt2d.commons.accession.persistence.mongodb.repository"}) -@EnableMongoAuditing -@AutoConfigureDataMongo -public class MongoTestConfiguration { - - @Value("${spring.data.mongodb.uri}") - private String mongoUri; - - @Value("${spring.data.mongodb.database}") - private String database; - - @Value("${mongodb.read-preference:primary}") - private String readPreference; - - @Bean - @Primary - public MongoClient mongoClient() { - return MongoClients.create(mongoUri); - } - - @Bean("primaryFactory") - @Primary - public MongoDatabaseFactory mongoDbFactory() { - return new SimpleMongoClientDatabaseFactory(mongoClient(), database); - } - - @Bean - @Primary - public MongoCustomConversions mongoCustomConversions() { - return new MongoCustomConversions(Collections.emptyList()); - } - - @Bean - @Primary - public MongoMappingContext mongoMappingContext(MongoCustomConversions conversions) { - MongoMappingContext context = new MongoMappingContext(); - context.setSimpleTypeHolder(conversions.getSimpleTypeHolder()); - context.setAutoIndexCreation(false); - return context; - } - - @Bean - @Primary - public MongoTemplate mongoTemplate(MongoMappingContext mongoMappingContext, - MongoCustomConversions conversions) { - MappingMongoConverter converter = new MappingMongoConverter( - new DefaultDbRefResolver(mongoDbFactory()), mongoMappingContext); - converter.setCustomConversions(conversions); - converter.setTypeMapper(new DefaultMongoTypeMapper(null)); - converter.afterPropertiesSet(); - MongoTemplate mongoTemplate = new MongoTemplate(mongoDbFactory(), converter); - mongoTemplate.setReadPreference(ReadPreference.valueOf(readPreference)); - mongoTemplate.setWriteConcern(WriteConcern.MAJORITY); - mongoTemplate.setWriteResultChecking(WriteResultChecking.EXCEPTION); - return mongoTemplate; - } -} diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/test/SSAccessionRecoveryTestConfiguration.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/test/SSAccessionRecoveryTestConfiguration.java index 4556a4439..0adc8b321 100644 --- a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/test/SSAccessionRecoveryTestConfiguration.java +++ b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/test/SSAccessionRecoveryTestConfiguration.java @@ -19,7 +19,6 @@ import org.springframework.batch.core.launch.JobLauncher; import org.springframework.batch.core.repository.JobRepository; import org.springframework.batch.test.JobLauncherTestUtils; -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Import; import uk.ac.ebi.eva.accession.pipeline.configuration.batch.io.AccessionWriterConfiguration; @@ -31,7 +30,6 @@ import uk.ac.ebi.eva.accession.pipeline.runner.EvaAccessionJobLauncherCommandLineRunner; import uk.ac.ebi.eva.commons.batch.job.JobExecutionApplicationListener; -@EnableAutoConfiguration @Import({SSAccessionRecoveryJobConfiguration.class, SSAccessionRecoveryStepConfiguration.class, SSAccessionRecoveryServiceConfiguration.class, diff --git a/eva-accession-pipeline/src/test/resources/properties/accession-pipeline-interval-test.properties b/eva-accession-pipeline/src/test/resources/properties/accession-pipeline-interval-test.properties index b30fb3297..3f84f1df4 100644 --- a/eva-accession-pipeline/src/test/resources/properties/accession-pipeline-interval-test.properties +++ b/eva-accession-pipeline/src/test/resources/properties/accession-pipeline-interval-test.properties @@ -4,6 +4,11 @@ accessioning.monotonic.test-pipeline-ss.blockSize=2 accessioning.monotonic.test-pipeline-ss.blockStartValue=500 accessioning.monotonic.test-pipeline-ss.nextBlockInterval=10 +continuous.id.blocks.datasource.driver-class-name=org.hsqldb.jdbc.JDBCDriver +continuous.id.blocks.datasource.jdbc-url=jdbc:hsqldb:mem:db;sql.syntax_pgs=true;DB_CLOSE_DELAY=-1 +continuous.id.blocks.datasource.username=SA +continuous.id.blocks.datasource.password= + eva.count-stats.url=http://localhost:8080 eva.count-stats.username=username eva.count-stats.password=password diff --git a/eva-accession-pipeline/src/test/resources/properties/accession-pipeline-recover-state-test.properties b/eva-accession-pipeline/src/test/resources/properties/accession-pipeline-recover-state-test.properties index 2edb00288..ff59b51ba 100644 --- a/eva-accession-pipeline/src/test/resources/properties/accession-pipeline-recover-state-test.properties +++ b/eva-accession-pipeline/src/test/resources/properties/accession-pipeline-recover-state-test.properties @@ -4,14 +4,10 @@ accessioning.monotonic.test-pipeline-recover-state-ss.blockSize=30 accessioning.monotonic.test-pipeline-recover-state-ss.blockStartValue=5000000000 accessioning.monotonic.test-pipeline-recover-state-ss.nextBlockInterval=1000000000 -spring.datasource.driver-class-name=org.hsqldb.jdbcDriver -spring.datasource.url=jdbc:hsqldb:mem:db;sql.syntax_pgs=true;DB_CLOSE_DELAY=-1 -spring.datasource.username=SA -spring.datasource.password= -spring.sql.init.schema-locations=classpath:test-data/contiguous_id_blocks_schema.sql -spring.sql.init.data-locations=classpath:test-data/contiguous_id_blocks_recover_state_data.sql -spring.sql.init.mode=always -spring.jpa.hibernate.ddl-auto=none +continuous.id.blocks.datasource.driver-class-name=org.hsqldb.jdbc.JDBCDriver +continuous.id.blocks.datasource.jdbc-url=jdbc:hsqldb:mem:db;sql.syntax_pgs=true;DB_CLOSE_DELAY=-1 +continuous.id.blocks.datasource.username=SA +continuous.id.blocks.datasource.password= parameters.assemblyAccession=assembly parameters.taxonomyAccession=1111 diff --git a/eva-accession-pipeline/src/test/resources/properties/accession-pipeline-recover-test.properties b/eva-accession-pipeline/src/test/resources/properties/accession-pipeline-recover-test.properties index 26d8215a4..de830075f 100644 --- a/eva-accession-pipeline/src/test/resources/properties/accession-pipeline-recover-test.properties +++ b/eva-accession-pipeline/src/test/resources/properties/accession-pipeline-recover-test.properties @@ -8,14 +8,12 @@ eva.count-stats.url=http://localhost:8080 eva.count-stats.username=username eva.count-stats.password=password -spring.datasource.driver-class-name=org.hsqldb.jdbcDriver -spring.datasource.url=jdbc:hsqldb:mem:db;sql.syntax_pgs=true;DB_CLOSE_DELAY=-1 -spring.datasource.username=SA -spring.datasource.password= -spring.sql.init.schema-locations=classpath:test-data/contiguous_id_blocks_schema.sql +continuous.id.blocks.datasource.driver-class-name=org.hsqldb.jdbc.JDBCDriver +continuous.id.blocks.datasource.jdbc-url=jdbc:hsqldb:mem:db;sql.syntax_pgs=true;DB_CLOSE_DELAY=-1 +continuous.id.blocks.datasource.username=SA +continuous.id.blocks.datasource.password= + spring.sql.init.data-locations=classpath:test-data/contiguous_id_blocks_data.sql -spring.sql.init.mode=always -spring.jpa.hibernate.ddl-auto=none parameters.assemblyAccession=assembly parameters.taxonomyAccession=1111 diff --git a/eva-accession-pipeline/src/test/resources/properties/accession-pipeline-test.properties b/eva-accession-pipeline/src/test/resources/properties/accession-pipeline-test.properties index 5a20cf25b..a503929da 100644 --- a/eva-accession-pipeline/src/test/resources/properties/accession-pipeline-test.properties +++ b/eva-accession-pipeline/src/test/resources/properties/accession-pipeline-test.properties @@ -5,6 +5,11 @@ eva.count-stats.url=http://localhost:8080 eva.count-stats.username=username eva.count-stats.password=password +continuous.id.blocks.datasource.driver-class-name=org.hsqldb.jdbc.JDBCDriver +continuous.id.blocks.datasource.jdbc-url=jdbc:hsqldb:mem:db;sql.syntax_pgs=true;DB_CLOSE_DELAY=-1 +continuous.id.blocks.datasource.username=SA +continuous.id.blocks.datasource.password= + accessioning.monotonic.test-pipeline-ss.blockSize=100000 accessioning.monotonic.test-pipeline-ss.blockStartValue=5000000000 accessioning.monotonic.test-pipeline-ss.nextBlockInterval=1000000000 diff --git a/eva-accession-pipeline/src/test/resources/properties/duplicate-ss-acc-qc-test.properties b/eva-accession-pipeline/src/test/resources/properties/duplicate-ss-acc-qc-test.properties index 183cdbf90..571b8e234 100644 --- a/eva-accession-pipeline/src/test/resources/properties/duplicate-ss-acc-qc-test.properties +++ b/eva-accession-pipeline/src/test/resources/properties/duplicate-ss-acc-qc-test.properties @@ -5,6 +5,11 @@ eva.count-stats.url=http://localhost:8080 eva.count-stats.username=username eva.count-stats.password=password +continuous.id.blocks.datasource.driver-class-name=org.hsqldb.jdbc.JDBCDriver +continuous.id.blocks.datasource.jdbc-url=jdbc:hsqldb:mem:db;sql.syntax_pgs=true;DB_CLOSE_DELAY=-1 +continuous.id.blocks.datasource.username=SA +continuous.id.blocks.datasource.password= + accessioning.monotonic.test-pipeline-ss.blockSize=100000 accessioning.monotonic.test-pipeline-ss.blockStartValue=5000000000 accessioning.monotonic.test-pipeline-ss.nextBlockInterval=1000000000 diff --git a/eva-accession-pipeline/src/test/resources/properties/qc-accession-pipeline-test.properties b/eva-accession-pipeline/src/test/resources/properties/qc-accession-pipeline-test.properties index 28c3df942..398a465b7 100644 --- a/eva-accession-pipeline/src/test/resources/properties/qc-accession-pipeline-test.properties +++ b/eva-accession-pipeline/src/test/resources/properties/qc-accession-pipeline-test.properties @@ -8,6 +8,11 @@ accessioning.monotonic.test-pipeline-ss.blockSize=100000 accessioning.monotonic.test-pipeline-ss.blockStartValue=5000000000 accessioning.monotonic.test-pipeline-ss.nextBlockInterval=1000000000 +continuous.id.blocks.datasource.driver-class-name=org.hsqldb.jdbc.JDBCDriver +continuous.id.blocks.datasource.jdbc-url=jdbc:hsqldb:mem:db;sql.syntax_pgs=true;DB_CLOSE_DELAY=-1 +continuous.id.blocks.datasource.username=SA +continuous.id.blocks.datasource.password= + parameters.assemblyAccession=assembly parameters.taxonomyAccession=1111 parameters.projectAccession=project diff --git a/eva-accession-pipeline/src/test/resources/properties/ss-accession-recovery.properties b/eva-accession-pipeline/src/test/resources/properties/ss-accession-recovery.properties index 343cf5fde..c651270b7 100644 --- a/eva-accession-pipeline/src/test/resources/properties/ss-accession-recovery.properties +++ b/eva-accession-pipeline/src/test/resources/properties/ss-accession-recovery.properties @@ -1,13 +1,9 @@ recovery.cutoff.days=14 -spring.datasource.driver-class-name=org.hsqldb.jdbcDriver -spring.datasource.url=jdbc:hsqldb:mem:db;sql.syntax_pgs=true;DB_CLOSE_DELAY=-1 -spring.datasource.username=SA -spring.datasource.password= -spring.sql.init.schema-locations=classpath:test-data/contiguous_id_blocks_schema.sql -spring.sql.init.data-locations=classpath:test-data/ss_accession_recovery_test_data.sql -spring.sql.init.mode=always -spring.jpa.hibernate.ddl-auto=none +continuous.id.blocks.datasource.driver-class-name=org.hsqldb.jdbc.JDBCDriver +continuous.id.blocks.datasource.jdbc-url=jdbc:hsqldb:mem:db;sql.syntax_pgs=true;DB_CLOSE_DELAY=-1 +continuous.id.blocks.datasource.username=SA +continuous.id.blocks.datasource.password= accessioning.submitted.categoryId=test-pipeline-ss diff --git a/eva-accession-pipeline/src/test/resources/test-data/contiguous_id_blocks_schema.sql b/eva-accession-pipeline/src/test/resources/test-data/contiguous_id_blocks_schema.sql index 495f9a4fb..32dfe369d 100644 --- a/eva-accession-pipeline/src/test/resources/test-data/contiguous_id_blocks_schema.sql +++ b/eva-accession-pipeline/src/test/resources/test-data/contiguous_id_blocks_schema.sql @@ -1,4 +1,8 @@ -CREATE TABLE contiguous_id_blocks ( +CREATE SEQUENCE IF NOT EXISTS contiguous_id_blocks_seq + START WITH 1 + INCREMENT BY 50; + +CREATE TABLE IF NOT EXISTS contiguous_id_blocks ( id bigint not NULL, application_instance_id varchar(255) not NULL, category_id varchar(255) not NULL, diff --git a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/Application.java b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/Application.java index 7c80c6fe7..86f86cfc1 100644 --- a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/Application.java +++ b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/Application.java @@ -15,20 +15,25 @@ */ package uk.ac.ebi.eva.accession.release; -import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.autoconfigure.batch.BatchAutoConfiguration; +import org.springframework.boot.autoconfigure.data.mongo.MongoDataAutoConfiguration; import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration; import org.springframework.boot.autoconfigure.jdbc.DataSourceTransactionManagerAutoConfiguration; import org.springframework.boot.autoconfigure.orm.jpa.HibernateJpaAutoConfiguration; import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.context.annotation.Import; +import uk.ac.ebi.eva.accession.core.configuration.InMemoryBatchConfiguration; -@EnableBatchProcessing @SpringBootApplication(exclude = { + MongoDataAutoConfiguration.class, DataSourceAutoConfiguration.class, DataSourceTransactionManagerAutoConfiguration.class, - HibernateJpaAutoConfiguration.class + HibernateJpaAutoConfiguration.class, + BatchAutoConfiguration.class }) +@Import(InMemoryBatchConfiguration.class) public class Application { public static void main(String[] args) { diff --git a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/RunnerConfiguration.java b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/RunnerConfiguration.java index a0c05b893..1758f27fc 100644 --- a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/RunnerConfiguration.java +++ b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/RunnerConfiguration.java @@ -16,15 +16,40 @@ */ package uk.ac.ebi.eva.accession.release.configuration; +import org.springframework.batch.core.ExitStatus; +import org.springframework.batch.core.Job; +import org.springframework.batch.core.JobExecution; +import org.springframework.batch.core.JobParametersBuilder; +import org.springframework.batch.core.launch.JobLauncher; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.CommandLineRunner; +import org.springframework.context.ApplicationContext; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; +import uk.ac.ebi.eva.accession.core.configuration.nonhuman.MongoConfiguration; import uk.ac.ebi.eva.commons.batch.job.JobExecutionApplicationListener; @Configuration +@Import({MongoConfiguration.class}) public class RunnerConfiguration { @Bean public JobExecutionApplicationListener jobExecutionApplicationListener() { return new JobExecutionApplicationListener(); } + + @Bean + public CommandLineRunner runJob(JobLauncher jobLauncher, ApplicationContext context, + @Value("${spring.batch.job.names}") String jobName) { + return args -> { + Job job = context.getBean(jobName, Job.class); + JobExecution execution = jobLauncher.run(job, new JobParametersBuilder() + .toJobParameters()); + + if (!execution.getExitStatus().equals(ExitStatus.COMPLETED)) { + throw new RuntimeException("Job failed with status: " + execution.getExitStatus()); + } + }; + } } \ No newline at end of file diff --git a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/jobs/ActiveAccessionReleaseFromDBJobConfiguration.java b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/jobs/ActiveAccessionReleaseFromDBJobConfiguration.java index fe595823c..c53f497fd 100644 --- a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/jobs/ActiveAccessionReleaseFromDBJobConfiguration.java +++ b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/jobs/ActiveAccessionReleaseFromDBJobConfiguration.java @@ -2,9 +2,7 @@ import org.springframework.batch.core.Job; import org.springframework.batch.core.Step; -import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.batch.core.job.builder.JobBuilder; -import org.springframework.batch.core.launch.support.RunIdIncrementer; import org.springframework.batch.core.repository.JobRepository; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; @@ -15,7 +13,6 @@ import static uk.ac.ebi.eva.accession.release.configuration.BeanNames.ACTIVE_ACCESSIONS_RELEASE_FROM_DB_STEP; @Configuration -@EnableBatchProcessing public class ActiveAccessionReleaseFromDBJobConfiguration { @Autowired @@ -25,7 +22,6 @@ public class ActiveAccessionReleaseFromDBJobConfiguration { @Bean(ACTIVE_ACCESSIONS_RELEASE_FROM_DB_JOB) public Job activeAccessionReleaseFromDBJob(JobRepository jobRepository) { return new JobBuilder(ACTIVE_ACCESSIONS_RELEASE_FROM_DB_JOB, jobRepository) - .incrementer(new RunIdIncrementer()) .start(activeAccessionsReleaseFromDBStep) .build(); } diff --git a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/jobs/DumpRSAccessionsJobConfiguration.java b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/jobs/DumpRSAccessionsJobConfiguration.java index e8be4c938..a81c2c452 100644 --- a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/jobs/DumpRSAccessionsJobConfiguration.java +++ b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/jobs/DumpRSAccessionsJobConfiguration.java @@ -3,9 +3,7 @@ import org.springframework.batch.core.Job; import org.springframework.batch.core.Step; -import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.batch.core.job.builder.JobBuilder; -import org.springframework.batch.core.launch.support.RunIdIncrementer; import org.springframework.batch.core.repository.JobRepository; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; @@ -18,7 +16,6 @@ import static uk.ac.ebi.eva.accession.release.configuration.BeanNames.DUMP_MERGED_AND_DEPRECATED_ACCESSIONS_STEP; @Configuration -@EnableBatchProcessing public class DumpRSAccessionsJobConfiguration { @Autowired @@ -32,7 +29,6 @@ public class DumpRSAccessionsJobConfiguration { @Bean(DUMP_ACTIVE_ACCESSIONS_JOB) public Job dumpActiveAccessionJob(JobRepository jobRepository) { return new JobBuilder(DUMP_ACTIVE_ACCESSIONS_JOB, jobRepository) - .incrementer(new RunIdIncrementer()) .start(dumpActiveAccessionsStep) .build(); } @@ -40,7 +36,6 @@ public Job dumpActiveAccessionJob(JobRepository jobRepository) { @Bean(DUMP_MERGED_AND_DEPRECATED_ACCESSIONS_JOB) public Job dumpMergedAndDeprecatedAccessionJob(JobRepository jobRepository) { return new JobBuilder(DUMP_MERGED_AND_DEPRECATED_ACCESSIONS_JOB, jobRepository) - .incrementer(new RunIdIncrementer()) .start(dumpMergedAndDeprecatedAccessionsStep) .build(); } diff --git a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/jobs/MergedAndDeprecatedAccessionReleaseFromDBJobConfiguration.java b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/jobs/MergedAndDeprecatedAccessionReleaseFromDBJobConfiguration.java index acf2c36b6..d559d3abd 100644 --- a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/jobs/MergedAndDeprecatedAccessionReleaseFromDBJobConfiguration.java +++ b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/jobs/MergedAndDeprecatedAccessionReleaseFromDBJobConfiguration.java @@ -2,9 +2,7 @@ import org.springframework.batch.core.Job; import org.springframework.batch.core.Step; -import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.batch.core.job.builder.JobBuilder; -import org.springframework.batch.core.launch.support.RunIdIncrementer; import org.springframework.batch.core.repository.JobRepository; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; @@ -15,7 +13,6 @@ import static uk.ac.ebi.eva.accession.release.configuration.BeanNames.MERGED_AND_DEPRECATED_ACCESSIONS_RELEASE_FROM_DB_STEP; @Configuration -@EnableBatchProcessing public class MergedAndDeprecatedAccessionReleaseFromDBJobConfiguration { @Autowired @@ -25,7 +22,6 @@ public class MergedAndDeprecatedAccessionReleaseFromDBJobConfiguration { @Bean(MERGED_AND_DEPRECATED_ACCESSIONS_RELEASE_FROM_DB_JOB) public Job mergedAndDeprecatedAccessionReleaseFromDBJob(JobRepository jobRepository) { return new JobBuilder(MERGED_AND_DEPRECATED_ACCESSIONS_RELEASE_FROM_DB_JOB, jobRepository) - .incrementer(new RunIdIncrementer()) .start(mergedAndDeprecatedAccessionsReleaseFromDBStep) .build(); } diff --git a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/steps/ActiveAccessionReleaseFromDBStepConfiguration.java b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/steps/ActiveAccessionReleaseFromDBStepConfiguration.java index be3a1e5d5..7bbb10618 100644 --- a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/steps/ActiveAccessionReleaseFromDBStepConfiguration.java +++ b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/steps/ActiveAccessionReleaseFromDBStepConfiguration.java @@ -2,7 +2,6 @@ import htsjdk.variant.variantcontext.VariantContext; import org.springframework.batch.core.Step; -import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.batch.core.repository.JobRepository; import org.springframework.batch.core.step.builder.StepBuilder; import org.springframework.batch.core.step.tasklet.TaskletStep; @@ -17,13 +16,13 @@ import org.springframework.transaction.PlatformTransactionManager; import uk.ac.ebi.eva.commons.core.models.pipeline.Variant; +import static uk.ac.ebi.eva.accession.core.configuration.InMemoryBatchConfiguration.BATCH_TRANSACTION_MANAGER; import static uk.ac.ebi.eva.accession.release.configuration.BeanNames.ACTIVE_ACCESSIONS_RELEASE_FROM_DB_STEP; import static uk.ac.ebi.eva.accession.release.configuration.BeanNames.ACTIVE_ACCESSIONS_VARIANT_READER; import static uk.ac.ebi.eva.accession.release.configuration.BeanNames.EVA_RELEASE_WRITER; import static uk.ac.ebi.eva.accession.release.configuration.BeanNames.RELEASE_PROCESSOR; @Configuration -@EnableBatchProcessing public class ActiveAccessionReleaseFromDBStepConfiguration { @Autowired @@ -39,7 +38,9 @@ public class ActiveAccessionReleaseFromDBStepConfiguration { ItemStreamWriter accessionWriter; @Bean(ACTIVE_ACCESSIONS_RELEASE_FROM_DB_STEP) - public Step activeAccessionsReleaseFromDBStep(JobRepository jobRepository, PlatformTransactionManager transactionManager, + public Step activeAccessionsReleaseFromDBStep(JobRepository jobRepository, + @Qualifier(BATCH_TRANSACTION_MANAGER) + PlatformTransactionManager transactionManager, SimpleCompletionPolicy chunkSizeCompletionPolicy) { TaskletStep step = new StepBuilder(ACTIVE_ACCESSIONS_RELEASE_FROM_DB_STEP, jobRepository) .chunk(chunkSizeCompletionPolicy, transactionManager) diff --git a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/steps/DumpRSAccessionsStepConfiguration.java b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/steps/DumpRSAccessionsStepConfiguration.java index 762b22ac1..ed27062c8 100644 --- a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/steps/DumpRSAccessionsStepConfiguration.java +++ b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/steps/DumpRSAccessionsStepConfiguration.java @@ -1,7 +1,6 @@ package uk.ac.ebi.eva.accession.release.configuration.batch.steps; import org.springframework.batch.core.Step; -import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.batch.core.repository.JobRepository; import org.springframework.batch.core.step.builder.StepBuilder; import org.springframework.beans.factory.annotation.Autowired; @@ -12,12 +11,12 @@ import uk.ac.ebi.eva.accession.release.batch.io.DumpRSAccessionsInFile; import uk.ac.ebi.eva.accession.release.parameters.InputParameters; +import static uk.ac.ebi.eva.accession.core.configuration.InMemoryBatchConfiguration.BATCH_TRANSACTION_MANAGER; import static uk.ac.ebi.eva.accession.release.configuration.BeanNames.DUMP_ACTIVE_ACCESSIONS_STEP; import static uk.ac.ebi.eva.accession.release.configuration.BeanNames.DUMP_MERGED_AND_DEPRECATED_ACCESSIONS_STEP; import static uk.ac.ebi.eva.accession.release.configuration.BeanNames.DUMP_RS_ACCESSIONS_IN_FILE; @Configuration -@EnableBatchProcessing public class DumpRSAccessionsStepConfiguration { @Autowired @@ -25,7 +24,8 @@ public class DumpRSAccessionsStepConfiguration { private DumpRSAccessionsInFile dumpRSAccessionsInFile; @Bean(DUMP_ACTIVE_ACCESSIONS_STEP) - public Step dumpActiveAccessionStep(JobRepository jobRepository, PlatformTransactionManager transactionManager, + public Step dumpActiveAccessionStep(JobRepository jobRepository, + @Qualifier(BATCH_TRANSACTION_MANAGER) PlatformTransactionManager transactionManager, InputParameters inputParameters) { return new StepBuilder(DUMP_ACTIVE_ACCESSIONS_STEP, jobRepository) .tasklet((contribution, chunkContext) -> { @@ -37,7 +37,8 @@ public Step dumpActiveAccessionStep(JobRepository jobRepository, PlatformTransac } @Bean(DUMP_MERGED_AND_DEPRECATED_ACCESSIONS_STEP) - public Step dumpMergedAndDeprecatedAccessionStep(JobRepository jobRepository, PlatformTransactionManager transactionManager, + public Step dumpMergedAndDeprecatedAccessionStep(JobRepository jobRepository, + @Qualifier(BATCH_TRANSACTION_MANAGER) PlatformTransactionManager transactionManager, InputParameters inputParameters) { return new StepBuilder(DUMP_MERGED_AND_DEPRECATED_ACCESSIONS_STEP, jobRepository) .tasklet((contribution, chunkContext) -> { diff --git a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/steps/MergedAndDeprecatedAccessionReleaseFromDBStepConfiguration.java b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/steps/MergedAndDeprecatedAccessionReleaseFromDBStepConfiguration.java index a07221a2a..4dd0ec12d 100644 --- a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/steps/MergedAndDeprecatedAccessionReleaseFromDBStepConfiguration.java +++ b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/steps/MergedAndDeprecatedAccessionReleaseFromDBStepConfiguration.java @@ -2,7 +2,6 @@ import htsjdk.variant.variantcontext.VariantContext; import org.springframework.batch.core.Step; -import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.batch.core.repository.JobRepository; import org.springframework.batch.core.step.builder.StepBuilder; import org.springframework.batch.core.step.tasklet.TaskletStep; @@ -17,13 +16,13 @@ import org.springframework.transaction.PlatformTransactionManager; import uk.ac.ebi.eva.commons.core.models.pipeline.Variant; +import static uk.ac.ebi.eva.accession.core.configuration.InMemoryBatchConfiguration.BATCH_TRANSACTION_MANAGER; import static uk.ac.ebi.eva.accession.release.configuration.BeanNames.EVA_MERGED_RELEASE_WRITER; import static uk.ac.ebi.eva.accession.release.configuration.BeanNames.MERGED_AND_DEPRECATED_ACCESSIONS_RELEASE_FROM_DB_STEP; import static uk.ac.ebi.eva.accession.release.configuration.BeanNames.MERGED_AND_DEPRECATED_ACCESSIONS_VARIANT_READER; import static uk.ac.ebi.eva.accession.release.configuration.BeanNames.RELEASE_PROCESSOR; @Configuration -@EnableBatchProcessing public class MergedAndDeprecatedAccessionReleaseFromDBStepConfiguration { @Autowired @@ -39,7 +38,9 @@ public class MergedAndDeprecatedAccessionReleaseFromDBStepConfiguration { ItemStreamWriter accessionWriter; @Bean(MERGED_AND_DEPRECATED_ACCESSIONS_RELEASE_FROM_DB_STEP) - public Step mergedAndDeprecatedAccessionsReleaseFromDBStep(JobRepository jobRepository, PlatformTransactionManager transactionManager, + public Step mergedAndDeprecatedAccessionsReleaseFromDBStep(JobRepository jobRepository, + @Qualifier(BATCH_TRANSACTION_MANAGER) + PlatformTransactionManager transactionManager, SimpleCompletionPolicy chunkSizeCompletionPolicy) { TaskletStep step = new StepBuilder(MERGED_AND_DEPRECATED_ACCESSIONS_RELEASE_FROM_DB_STEP, jobRepository) .chunk(chunkSizeCompletionPolicy, transactionManager) diff --git a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/parameters/InputParameters.java b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/parameters/InputParameters.java index 832e6fdfe..d17f6febc 100644 --- a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/parameters/InputParameters.java +++ b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/parameters/InputParameters.java @@ -33,8 +33,6 @@ public class InputParameters { private String outputFolder; - private boolean forceRestart; - private int chunkSize; private String accessionedVcf; @@ -97,14 +95,6 @@ public void setOutputFolder(String outputFolder) { this.outputFolder = outputFolder; } - public boolean isForceRestart() { - return forceRestart; - } - - public void setForceRestart(boolean forceRestart) { - this.forceRestart = forceRestart; - } - public int getChunkSize() { return chunkSize; } diff --git a/eva-accession-release/src/main/resources/application.properties b/eva-accession-release/src/main/resources/application.properties index 65cf102b2..0b1c7a873 100644 --- a/eva-accession-release/src/main/resources/application.properties +++ b/eva-accession-release/src/main/resources/application.properties @@ -5,7 +5,6 @@ parameters.taxonomyAccession= parameters.fasta= parameters.assemblyReportUrl= parameters.outputFolder= -parameters.forceRestart=false parameters.chunkSize=100 parameters.contigNaming=SEQUENCE_NAME diff --git a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/active/AccessionedVariantMongoReaderTest.java b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/active/AccessionedVariantMongoReaderTest.java index 14bb1febe..6a0278069 100644 --- a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/active/AccessionedVariantMongoReaderTest.java +++ b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/active/AccessionedVariantMongoReaderTest.java @@ -33,10 +33,10 @@ import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.MongoConfiguration; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.core.utils.MongoTestDataLoader; import uk.ac.ebi.eva.accession.release.collectionNames.DbsnpCollectionNames; -import uk.ac.ebi.eva.accession.release.test.configuration.MongoTestConfiguration; import uk.ac.ebi.eva.commons.core.models.pipeline.Variant; import java.util.ArrayList; diff --git a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/contig/ContigMongoReaderTest.java b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/contig/ContigMongoReaderTest.java index 17056b906..7df1ce539 100644 --- a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/contig/ContigMongoReaderTest.java +++ b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/contig/ContigMongoReaderTest.java @@ -29,11 +29,11 @@ import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.MongoConfiguration; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.core.utils.MongoTestDataLoader; import uk.ac.ebi.eva.accession.release.collectionNames.DbsnpCollectionNames; import uk.ac.ebi.eva.accession.release.collectionNames.EvaCollectionNames; -import uk.ac.ebi.eva.accession.release.test.configuration.MongoTestConfiguration; import java.util.ArrayList; import java.util.Arrays; diff --git a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/deprecated/DeprecatedVariantMongoReaderTest.java b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/deprecated/DeprecatedVariantMongoReaderTest.java index 75adc3d34..f47e90bec 100644 --- a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/deprecated/DeprecatedVariantMongoReaderTest.java +++ b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/deprecated/DeprecatedVariantMongoReaderTest.java @@ -40,9 +40,9 @@ import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpSubmittedVariantOperationEntity; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; import uk.ac.ebi.eva.accession.core.summary.ClusteredVariantSummaryFunction; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.release.collectionNames.DbsnpCollectionNames; -import uk.ac.ebi.eva.accession.release.test.configuration.MongoTestConfiguration; import uk.ac.ebi.eva.commons.core.models.pipeline.Variant; import java.util.ArrayList; diff --git a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/merged/MergedVariantMongoReaderTest.java b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/merged/MergedVariantMongoReaderTest.java index 257d5bee5..aecdaef2e 100644 --- a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/merged/MergedVariantMongoReaderTest.java +++ b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/merged/MergedVariantMongoReaderTest.java @@ -43,11 +43,11 @@ import uk.ac.ebi.eva.accession.core.model.eva.ClusteredVariantEntity; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; import uk.ac.ebi.eva.accession.core.summary.ClusteredVariantSummaryFunction; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.core.utils.MongoTestDataLoader; import uk.ac.ebi.eva.accession.release.collectionNames.DbsnpCollectionNames; import uk.ac.ebi.eva.accession.release.collectionNames.EvaCollectionNames; -import uk.ac.ebi.eva.accession.release.test.configuration.MongoTestConfiguration; import uk.ac.ebi.eva.commons.core.models.pipeline.Variant; import java.util.ArrayList; diff --git a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/merged_deprecated/MergedDeprecatedVariantMongoReaderTest.java b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/merged_deprecated/MergedDeprecatedVariantMongoReaderTest.java index ef211280c..20803bc22 100644 --- a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/merged_deprecated/MergedDeprecatedVariantMongoReaderTest.java +++ b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/merged_deprecated/MergedDeprecatedVariantMongoReaderTest.java @@ -31,10 +31,10 @@ import uk.ac.ebi.ampt2d.commons.accession.core.models.EventType; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.MongoConfiguration; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpClusteredVariantOperationEntity; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.core.utils.MongoTestDataLoader; import uk.ac.ebi.eva.accession.release.collectionNames.DbsnpCollectionNames; -import uk.ac.ebi.eva.accession.release.test.configuration.MongoTestConfiguration; import java.util.ArrayList; import java.util.List; diff --git a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/configuration/batch/steps/DumpRSAccessionsTest.java b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/configuration/batch/steps/DumpRSAccessionsTest.java index 5308c0b55..eb65f6f96 100644 --- a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/configuration/batch/steps/DumpRSAccessionsTest.java +++ b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/configuration/batch/steps/DumpRSAccessionsTest.java @@ -30,15 +30,15 @@ import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.ampt2d.commons.accession.core.models.EventType; +import uk.ac.ebi.eva.accession.core.configuration.InMemoryBatchConfiguration; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpClusteredVariantEntity; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpClusteredVariantInactiveEntity; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpClusteredVariantOperationEntity; import uk.ac.ebi.eva.accession.core.model.eva.ClusteredVariantEntity; import uk.ac.ebi.eva.accession.core.model.eva.ClusteredVariantInactiveEntity; import uk.ac.ebi.eva.accession.core.model.eva.ClusteredVariantOperationEntity; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; -import uk.ac.ebi.eva.accession.release.test.configuration.BatchJobRepositoryTestConfiguration; -import uk.ac.ebi.eva.accession.release.test.configuration.MongoTestConfiguration; import uk.ac.ebi.eva.accession.release.test.configuration.ReleaseFromDBTestConfiguration; import uk.ac.ebi.eva.commons.core.models.VariantType; @@ -58,8 +58,8 @@ import static uk.ac.ebi.eva.accession.release.test.configuration.ReleaseFromDBTestConfiguration.TEST_DUMP_MERGED_AND_DEPRECATED_ACCESSIONS_JOB; @ExtendWith(SpringExtension.class) -@ContextConfiguration(classes = {MongoTestConfiguration.class, ReleaseFromDBTestConfiguration.class, - BatchJobRepositoryTestConfiguration.class}) +@ContextConfiguration(classes = {InMemoryBatchConfiguration.class, MongoTestConfiguration.class, + ReleaseFromDBTestConfiguration.class}) @TestPropertySource("classpath:dump-rs-accession-test.properties") public class DumpRSAccessionsTest extends MongoTestContainerHelper { private static final String accDumpFile = "src/test/resources/accDumpFile.csv"; diff --git a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/configuration/batch/steps/RSAccessionsReleaseFromDBTest.java b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/configuration/batch/steps/RSAccessionsReleaseFromDBTest.java index 527f11039..737fc9ca0 100644 --- a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/configuration/batch/steps/RSAccessionsReleaseFromDBTest.java +++ b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/configuration/batch/steps/RSAccessionsReleaseFromDBTest.java @@ -30,6 +30,7 @@ import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.ampt2d.commons.accession.core.models.EventType; +import uk.ac.ebi.eva.accession.core.configuration.InMemoryBatchConfiguration; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpClusteredVariantEntity; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpSubmittedVariantEntity; import uk.ac.ebi.eva.accession.core.model.eva.ClusteredVariantEntity; @@ -38,11 +39,10 @@ import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantInactiveEntity; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantOperationEntity; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.release.parameters.InputParameters; import uk.ac.ebi.eva.accession.release.parameters.ReportPathResolver; -import uk.ac.ebi.eva.accession.release.test.configuration.BatchJobRepositoryTestConfiguration; -import uk.ac.ebi.eva.accession.release.test.configuration.MongoTestConfiguration; import uk.ac.ebi.eva.accession.release.test.configuration.ReleaseFromDBTestConfiguration; import uk.ac.ebi.eva.commons.core.models.VariantType; import uk.ac.ebi.eva.commons.core.utils.FileUtils; @@ -71,8 +71,8 @@ import static uk.ac.ebi.eva.accession.release.test.configuration.ReleaseFromDBTestConfiguration.TEST_RELEASE_MERGED_AND_DEPRECATED_ACCESSIONS_JOB; @ExtendWith(SpringExtension.class) -@ContextConfiguration(classes = {MongoTestConfiguration.class, ReleaseFromDBTestConfiguration.class, - BatchJobRepositoryTestConfiguration.class}) +@ContextConfiguration(classes = {InMemoryBatchConfiguration.class, MongoTestConfiguration.class, + ReleaseFromDBTestConfiguration.class}) @TestPropertySource("classpath:dump-rs-accession-test.properties") public class RSAccessionsReleaseFromDBTest extends MongoTestContainerHelper { private static final String testRunDir = "src/test/resources/release-test-run"; diff --git a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/test/configuration/BatchJobRepositoryTestConfiguration.java b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/test/configuration/BatchJobRepositoryTestConfiguration.java deleted file mode 100644 index a4ef0782d..000000000 --- a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/test/configuration/BatchJobRepositoryTestConfiguration.java +++ /dev/null @@ -1,22 +0,0 @@ -package uk.ac.ebi.eva.accession.release.test.configuration; - -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Primary; -import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseBuilder; -import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseType; - -import javax.sql.DataSource; - -@Configuration -public class BatchJobRepositoryTestConfiguration { - - @Bean - @Primary - public DataSource dataSource() { - return new EmbeddedDatabaseBuilder() - .setType(EmbeddedDatabaseType.HSQL) - .addScript("org/springframework/batch/core/schema-hsqldb.sql") - .build(); - } -} \ No newline at end of file diff --git a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/test/configuration/BatchTestConfiguration.java b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/test/configuration/BatchTestConfiguration.java index c90817cb3..3181faf7b 100644 --- a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/test/configuration/BatchTestConfiguration.java +++ b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/test/configuration/BatchTestConfiguration.java @@ -17,7 +17,6 @@ package uk.ac.ebi.eva.accession.release.test.configuration; import org.springframework.batch.test.JobLauncherTestUtils; -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Import; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.MongoConfiguration; @@ -26,7 +25,6 @@ import uk.ac.ebi.eva.accession.release.configuration.batch.policies.IllegalStartSkipPolicyConfiguration; import uk.ac.ebi.eva.commons.batch.job.JobExecutionApplicationListener; -@EnableAutoConfiguration @Import({MongoConfiguration.class, InputParametersConfiguration.class, ChunkSizeCompletionPolicyConfiguration.class, diff --git a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/test/configuration/MongoTestConfiguration.java b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/test/configuration/MongoTestConfiguration.java deleted file mode 100644 index 3051afd22..000000000 --- a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/test/configuration/MongoTestConfiguration.java +++ /dev/null @@ -1,105 +0,0 @@ -/* - * - * Copyright 2018 EMBL - European Bioinformatics Institute - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -package uk.ac.ebi.eva.accession.release.test.configuration; - -import com.mongodb.ReadPreference; -import com.mongodb.WriteConcern; -import com.mongodb.client.MongoClient; -import com.mongodb.client.MongoClients; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.domain.EntityScan; -import org.springframework.boot.test.autoconfigure.data.mongo.AutoConfigureDataMongo; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Primary; -import org.springframework.data.mongodb.MongoDatabaseFactory; -import org.springframework.data.mongodb.config.EnableMongoAuditing; -import org.springframework.data.mongodb.core.MongoTemplate; -import org.springframework.data.mongodb.core.SimpleMongoClientDatabaseFactory; -import org.springframework.data.mongodb.core.WriteResultChecking; -import org.springframework.data.mongodb.core.convert.DefaultDbRefResolver; -import org.springframework.data.mongodb.core.convert.DefaultMongoTypeMapper; -import org.springframework.data.mongodb.core.convert.MappingMongoConverter; -import org.springframework.data.mongodb.core.convert.MongoCustomConversions; -import org.springframework.data.mongodb.core.mapping.MongoMappingContext; -import org.springframework.data.mongodb.repository.config.EnableMongoRepositories; - -import java.util.Collections; - -@Configuration -@EntityScan(basePackages = {"uk.ac.ebi.eva.accession.core.persistence"}) -@EnableMongoRepositories(basePackages = { - "uk.ac.ebi.eva.accession.core.persistence", - "uk.ac.ebi.ampt2d.commons.accession.persistence.mongodb.repository"}) -@EnableMongoAuditing -@AutoConfigureDataMongo -public class MongoTestConfiguration { - - @Value("${spring.data.mongodb.uri}") - private String mongoUri; - - @Value("${spring.data.mongodb.database}") - private String database; - - @Value("${mongodb.read-preference:primary}") - private String readPreference; - - @Bean - @Primary - public MongoClient mongoClient() { - return MongoClients.create(mongoUri); - } - - @Bean("primaryFactory") - @Primary - public MongoDatabaseFactory mongoDbFactory() { - return new SimpleMongoClientDatabaseFactory(mongoClient(), database); - } - - @Bean - @Primary - public MongoCustomConversions mongoCustomConversions() { - return new MongoCustomConversions(Collections.emptyList()); - } - - @Bean - @Primary - public MongoMappingContext mongoMappingContext(MongoCustomConversions conversions) { - MongoMappingContext context = new MongoMappingContext(); - context.setSimpleTypeHolder(conversions.getSimpleTypeHolder()); - context.setAutoIndexCreation(false); - return context; - } - - @Bean - @Primary - public MongoTemplate mongoTemplate(MongoMappingContext mongoMappingContext, - MongoCustomConversions conversions) { - MappingMongoConverter converter = new MappingMongoConverter( - new DefaultDbRefResolver(mongoDbFactory()), mongoMappingContext); - converter.setCustomConversions(conversions); - converter.setTypeMapper(new DefaultMongoTypeMapper(null)); - converter.afterPropertiesSet(); - MongoTemplate mongoTemplate = new MongoTemplate(mongoDbFactory(), converter); - mongoTemplate.setReadPreference(ReadPreference.valueOf(readPreference)); - mongoTemplate.setWriteConcern(WriteConcern.MAJORITY); - mongoTemplate.setWriteResultChecking(WriteResultChecking.EXCEPTION); - return mongoTemplate; - } - -} diff --git a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/test/configuration/ReleaseFromDBTestConfiguration.java b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/test/configuration/ReleaseFromDBTestConfiguration.java index 8f84a0cf6..ca43e079d 100644 --- a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/test/configuration/ReleaseFromDBTestConfiguration.java +++ b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/test/configuration/ReleaseFromDBTestConfiguration.java @@ -5,7 +5,6 @@ import org.springframework.batch.core.repository.JobRepository; import org.springframework.batch.test.JobLauncherTestUtils; import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Import; import uk.ac.ebi.eva.accession.release.configuration.InputParametersConfiguration; @@ -29,8 +28,6 @@ import static uk.ac.ebi.eva.accession.release.configuration.BeanNames.DUMP_MERGED_AND_DEPRECATED_ACCESSIONS_JOB; import static uk.ac.ebi.eva.accession.release.configuration.BeanNames.MERGED_AND_DEPRECATED_ACCESSIONS_RELEASE_FROM_DB_JOB; - -@EnableAutoConfiguration @Import({ChunkSizeCompletionPolicyConfiguration.class, IllegalStartSkipPolicyConfiguration.class, InputParametersConfiguration.class, diff --git a/eva-accession-release/src/test/resources/application.properties b/eva-accession-release/src/test/resources/application.properties index b8fb0bd6e..1aff6f39c 100644 --- a/eva-accession-release/src/test/resources/application.properties +++ b/eva-accession-release/src/test/resources/application.properties @@ -12,7 +12,6 @@ parameters.taxonomyAccession=60711 parameters.fasta=src/test/resources/input-files/fasta/GCF_000409795.2.test.fna parameters.assemblyReportUrl=file:src/test/resources/input-files/assembly-report/GCF_000409795.2_Chlorocebus_sabeus_1.1_assembly_report.txt parameters.outputFolder=/tmp/ -parameters.forceRestart=false parameters.chunkSize=1000 parameters.contigNaming=SEQUENCE_NAME diff --git a/eva-accession-release/src/test/resources/dump-rs-accession-test.properties b/eva-accession-release/src/test/resources/dump-rs-accession-test.properties index ee7516341..beec7f25e 100644 --- a/eva-accession-release/src/test/resources/dump-rs-accession-test.properties +++ b/eva-accession-release/src/test/resources/dump-rs-accession-test.properties @@ -12,7 +12,6 @@ parameters.taxonomyAccession=60711 parameters.fasta=src/test/resources/input-files/fasta/GCF_000409795.2.test.fna parameters.assemblyReportUrl=file:src/test/resources/input-files/assembly-report/GCF_000409795.2_Chlorocebus_sabeus_1.1_assembly_report.txt parameters.outputFolder=src/test/resources/release-test-run -parameters.forceRestart=false parameters.chunkSize=5 parameters.contigNaming=SEQUENCE_NAME diff --git a/eva-accession-release/src/test/resources/incremental-release-pipeline-test.properties b/eva-accession-release/src/test/resources/incremental-release-pipeline-test.properties deleted file mode 100644 index 70140cc36..000000000 --- a/eva-accession-release/src/test/resources/incremental-release-pipeline-test.properties +++ /dev/null @@ -1,27 +0,0 @@ -accessioning.instanceId=test-instance-01 -accessioning.submitted.categoryId=test-release-ss - -accessioning.monotonic.test-pipeline-ss.blockSize=100000 -accessioning.monotonic.test-pipeline-ss.blockStartValue=5000000000 -accessioning.monotonic.test-pipeline-ss.nextBlockInterval=1000000000 - -spring.jpa.show-sql=true - -parameters.assemblyAccession=GCA_000409795.2 -parameters.fasta=src/test/resources/input-files/fasta/GCF_000409795.2.test.fna -parameters.accessionedVcf=src/test/resources/test-data/incremental_release/study1.accessioned.vcf -parameters.assemblyReportUrl=file:src/test/resources/input-files/assembly-report/GCF_000409795.2_Chlorocebus_sabeus_1.1_assembly_report.txt -parameters.outputFolder=/tmp/ -parameters.forceRestart=false -parameters.chunkSize=5 -parameters.contigNaming=INSDC - -spring.data.mongodb.database=test-db -mongodb.read-preference=primary - -# See https://github.com/spring-projects/spring-boot/wiki/Spring-Boot-2.1-Release-Notes#bean-overriding -spring.main.allow-bean-definition-overriding=true - -# to fix exception javax.management.InstanceAlreadyExistsException: com.zaxxer.hikari:name=dataSource,type=HikariDataSource -# see https://stackoverflow.com/a/51798043/2375586 -spring.jmx.enabled=false diff --git a/eva-accession-release/src/test/resources/release-pipeline-test.properties b/eva-accession-release/src/test/resources/release-pipeline-test.properties deleted file mode 100644 index 48df3dc06..000000000 --- a/eva-accession-release/src/test/resources/release-pipeline-test.properties +++ /dev/null @@ -1,28 +0,0 @@ -accessioning.instanceId=test-instance-01 -accessioning.submitted.categoryId=test-release-ss - -accessioning.monotonic.test-pipeline-ss.blockSize=100000 -accessioning.monotonic.test-pipeline-ss.blockStartValue=5000000000 -accessioning.monotonic.test-pipeline-ss.nextBlockInterval=1000000000 - -spring.jpa.show-sql=true - -parameters.assemblyAccession=GCA_000409795.2 -parameters.taxonomyAccession=60711 -parameters.fasta=src/test/resources/input-files/fasta/GCF_000409795.2.test.fna -parameters.assemblyReportUrl=file:src/test/resources/input-files/assembly-report/GCF_000409795.2_Chlorocebus_sabeus_1.1_assembly_report.txt -parameters.outputFolder=/tmp/ -parameters.forceRestart=false -parameters.chunkSize=5 -parameters.contigNaming=SEQUENCE_NAME - - -spring.data.mongodb.database=test-db -mongodb.read-preference=primary - -# See https://github.com/spring-projects/spring-boot/wiki/Spring-Boot-2.1-Release-Notes#bean-overriding -spring.main.allow-bean-definition-overriding=true - -# to fix exception javax.management.InstanceAlreadyExistsException: com.zaxxer.hikari:name=dataSource,type=HikariDataSource -# see https://stackoverflow.com/a/51798043/2375586 -spring.jmx.enabled=false diff --git a/eva-accession-ws/pom.xml b/eva-accession-ws/pom.xml index a9e697f1c..897fdfe92 100644 --- a/eva-accession-ws/pom.xml +++ b/eva-accession-ws/pom.xml @@ -42,6 +42,11 @@ springdoc-openapi-starter-webmvc-ui + + org.postgresql + postgresql + + uk.ac.ebi.eva eva-accession-core @@ -53,6 +58,7 @@ org.hsqldb hsqldb + test diff --git a/eva-accession-ws/src/main/java/uk/ac/ebi/eva/accession/ws/EvaAccessionApplication.java b/eva-accession-ws/src/main/java/uk/ac/ebi/eva/accession/ws/EvaAccessionApplication.java index a4f3e3c30..100713ee1 100644 --- a/eva-accession-ws/src/main/java/uk/ac/ebi/eva/accession/ws/EvaAccessionApplication.java +++ b/eva-accession-ws/src/main/java/uk/ac/ebi/eva/accession/ws/EvaAccessionApplication.java @@ -19,11 +19,25 @@ import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.autoconfigure.batch.BatchAutoConfiguration; import org.springframework.boot.autoconfigure.data.mongo.MongoDataAutoConfiguration; +import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration; +import org.springframework.boot.autoconfigure.jdbc.DataSourceTransactionManagerAutoConfiguration; import org.springframework.boot.autoconfigure.mongo.MongoAutoConfiguration; +import org.springframework.boot.autoconfigure.orm.jpa.HibernateJpaAutoConfiguration; import org.springframework.boot.web.servlet.support.SpringBootServletInitializer; +import org.springframework.context.annotation.Import; +import uk.ac.ebi.eva.accession.core.configuration.ContiguousIdBlocksDataSourceConfiguration; -@SpringBootApplication(exclude = {MongoAutoConfiguration.class, MongoDataAutoConfiguration.class}) +@SpringBootApplication(exclude = { + MongoAutoConfiguration.class, + MongoDataAutoConfiguration.class, + DataSourceTransactionManagerAutoConfiguration.class, + DataSourceAutoConfiguration.class, + HibernateJpaAutoConfiguration.class, + BatchAutoConfiguration.class +}) +@Import({ContiguousIdBlocksDataSourceConfiguration.class}) public class EvaAccessionApplication extends SpringBootServletInitializer { public static void main(String[] args) { diff --git a/eva-accession-ws/src/main/java/uk/ac/ebi/eva/accession/ws/rest/EvaControllerAdvice.java b/eva-accession-ws/src/main/java/uk/ac/ebi/eva/accession/ws/rest/EvaControllerAdvice.java index d1c79691e..5aa1839ac 100644 --- a/eva-accession-ws/src/main/java/uk/ac/ebi/eva/accession/ws/rest/EvaControllerAdvice.java +++ b/eva-accession-ws/src/main/java/uk/ac/ebi/eva/accession/ws/rest/EvaControllerAdvice.java @@ -15,6 +15,7 @@ */ package uk.ac.ebi.eva.accession.ws.rest; +import jakarta.servlet.http.HttpServletRequest; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Value; @@ -29,7 +30,6 @@ import uk.ac.ebi.ampt2d.commons.accession.rest.BasicRestControllerAdvice; import uk.ac.ebi.ampt2d.commons.accession.rest.dto.ErrorMessage; -import jakarta.servlet.http.HttpServletRequest; import java.net.URI; /** diff --git a/eva-accession-ws/src/main/resources/application.properties b/eva-accession-ws/src/main/resources/application.properties index f524486de..cdb4e4b78 100644 --- a/eva-accession-ws/src/main/resources/application.properties +++ b/eva-accession-ws/src/main/resources/application.properties @@ -34,3 +34,8 @@ spring.main.allow-bean-definition-overriding=true spring.jmx.default-domain=eva.ebi.ac.uk.|timestamp| management.metrics.binders.jvm.enabled=false + +continuous.id.blocks.datasource.driver-class-name=org.postgresql.Driver +continuous.id.blocks.datasource.jdbc-url=|eva.accession.jdbc.ur| +continuous.id.blocks.datasource.username=|eva.accession.user| +continuous.id.blocks.datasource.password=|eva.accession.password| \ No newline at end of file diff --git a/eva-accession-ws/src/test/java/uk/ac/ebi/eva/accession/ws/ClusteredVariantHistoryEndPointTest.java b/eva-accession-ws/src/test/java/uk/ac/ebi/eva/accession/ws/ClusteredVariantHistoryEndPointTest.java index 9a716ab72..a3880aa25 100644 --- a/eva-accession-ws/src/test/java/uk/ac/ebi/eva/accession/ws/ClusteredVariantHistoryEndPointTest.java +++ b/eva-accession-ws/src/test/java/uk/ac/ebi/eva/accession/ws/ClusteredVariantHistoryEndPointTest.java @@ -37,13 +37,14 @@ import uk.ac.ebi.ampt2d.commons.accession.core.models.EventType; import uk.ac.ebi.ampt2d.commons.accession.rest.dto.AccessionResponseDTO; import uk.ac.ebi.ampt2d.commons.accession.rest.dto.HistoryEventDTO; +import uk.ac.ebi.eva.accession.core.configuration.ContiguousIdBlocksDataSourceConfiguration; import uk.ac.ebi.eva.accession.core.model.ClusteredVariant; import uk.ac.ebi.eva.accession.core.model.IClusteredVariant; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.core.utils.MongoTestDataLoader; import uk.ac.ebi.eva.accession.ws.dto.VariantHistory; import uk.ac.ebi.eva.accession.ws.rest.ClusteredVariantsRestController; -import uk.ac.ebi.eva.accession.ws.test.MongoTestConfiguration; import uk.ac.ebi.eva.commons.core.models.contigalias.ContigNamingConvention; import java.util.Arrays; @@ -54,7 +55,7 @@ @ExtendWith(SpringExtension.class) @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) -@Import({MongoTestConfiguration.class}) +@Import({MongoTestConfiguration.class, ContiguousIdBlocksDataSourceConfiguration.class}) @TestPropertySource("classpath:accession-ws-test.properties") public class ClusteredVariantHistoryEndPointTest extends MongoTestContainerHelper { diff --git a/eva-accession-ws/src/test/java/uk/ac/ebi/eva/accession/ws/ClusteredVariantsRestControllerTest.java b/eva-accession-ws/src/test/java/uk/ac/ebi/eva/accession/ws/ClusteredVariantsRestControllerTest.java index 3718a92b6..636c05d0e 100644 --- a/eva-accession-ws/src/test/java/uk/ac/ebi/eva/accession/ws/ClusteredVariantsRestControllerTest.java +++ b/eva-accession-ws/src/test/java/uk/ac/ebi/eva/accession/ws/ClusteredVariantsRestControllerTest.java @@ -49,6 +49,7 @@ import uk.ac.ebi.ampt2d.commons.accession.hashing.SHA1HashingFunction; import uk.ac.ebi.ampt2d.commons.accession.persistence.mongodb.document.AccessionedDocument; import uk.ac.ebi.ampt2d.commons.accession.rest.dto.AccessionResponseDTO; +import uk.ac.ebi.eva.accession.core.configuration.ContiguousIdBlocksDataSourceConfiguration; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.ClusteredVariantAccessioningConfiguration; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.SubmittedVariantAccessioningConfiguration; import uk.ac.ebi.eva.accession.core.contigalias.ContigAliasService; @@ -72,10 +73,10 @@ import uk.ac.ebi.eva.accession.core.service.nonhuman.dbsnp.DbsnpClusteredVariantMonotonicAccessioningService; import uk.ac.ebi.eva.accession.core.summary.ClusteredVariantSummaryFunction; import uk.ac.ebi.eva.accession.core.summary.SubmittedVariantSummaryFunction; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.ws.rest.ClusteredVariantsRestController; import uk.ac.ebi.eva.accession.ws.service.ClusteredVariantsBeaconService; -import uk.ac.ebi.eva.accession.ws.test.MongoTestConfiguration; import uk.ac.ebi.eva.accession.ws.test.NoContigTranslationArgumentMatcher; import uk.ac.ebi.eva.commons.beacon.models.BeaconAlleleRequest; import uk.ac.ebi.eva.commons.beacon.models.BeaconAlleleResponse; @@ -112,7 +113,7 @@ @ExtendWith(SpringExtension.class) @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) @Import({ClusteredVariantAccessioningConfiguration.class, SubmittedVariantAccessioningConfiguration.class, - MongoTestConfiguration.class}) + MongoTestConfiguration.class, ContiguousIdBlocksDataSourceConfiguration.class}) @TestPropertySource("classpath:accession-ws-test.properties") public class ClusteredVariantsRestControllerTest extends MongoTestContainerHelper { diff --git a/eva-accession-ws/src/test/java/uk/ac/ebi/eva/accession/ws/SubmittedVariantsRestControllerTest.java b/eva-accession-ws/src/test/java/uk/ac/ebi/eva/accession/ws/SubmittedVariantsRestControllerTest.java index eda734d86..1ee3a2307 100644 --- a/eva-accession-ws/src/test/java/uk/ac/ebi/eva/accession/ws/SubmittedVariantsRestControllerTest.java +++ b/eva-accession-ws/src/test/java/uk/ac/ebi/eva/accession/ws/SubmittedVariantsRestControllerTest.java @@ -30,12 +30,15 @@ import org.springframework.boot.test.web.client.TestRestTemplate; import org.springframework.context.annotation.Import; import org.springframework.core.ParameterizedTypeReference; +import org.springframework.core.io.ClassPathResource; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.http.HttpHeaders; import org.springframework.http.HttpMethod; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; +import org.springframework.jdbc.datasource.init.ResourceDatabasePopulator; import org.springframework.mock.web.MockHttpServletResponse; +import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit.jupiter.SpringExtension; import org.springframework.web.server.ResponseStatusException; @@ -47,6 +50,7 @@ import uk.ac.ebi.ampt2d.commons.accession.core.models.GetOrCreateAccessionWrapper; import uk.ac.ebi.ampt2d.commons.accession.rest.controllers.BasicRestController; import uk.ac.ebi.ampt2d.commons.accession.rest.dto.AccessionResponseDTO; +import uk.ac.ebi.eva.accession.core.configuration.ContiguousIdBlocksDataSourceConfiguration; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.SubmittedVariantAccessioningConfiguration; import uk.ac.ebi.eva.accession.core.contigalias.ContigAliasService; import uk.ac.ebi.eva.accession.core.model.ISubmittedVariant; @@ -67,6 +71,10 @@ import uk.ac.ebi.eva.accession.ws.test.NoContigTranslationArgumentMatcher; import uk.ac.ebi.eva.commons.core.models.contigalias.ContigNamingConvention; +import javax.sql.DataSource; +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -90,9 +98,10 @@ @ExtendWith(SpringExtension.class) @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) -@Import({SubmittedVariantAccessioningConfiguration.class}) +@Import({SubmittedVariantAccessioningConfiguration.class, ContiguousIdBlocksDataSourceConfiguration.class}) @TestPropertySource("classpath:accession-ws-test.properties") -public class SubmittedVariantsRestControllerTest extends MongoTestContainerHelper{ +@DirtiesContext(classMode = DirtiesContext.ClassMode.BEFORE_EACH_TEST_METHOD) +public class SubmittedVariantsRestControllerTest extends MongoTestContainerHelper { private static String TEST_APPLICATION_INSTANCE_ID = "test-application-instance-id"; private static final String URL = "/v1/submitted-variants/"; @@ -131,6 +140,9 @@ public class SubmittedVariantsRestControllerTest extends MongoTestContainerHelpe @MockBean private ContigAliasService contigAliasService; + @Autowired + private DataSource dataSource; + private List> generatedAccessions; private SubmittedVariant variant1; @@ -140,13 +152,21 @@ public class SubmittedVariantsRestControllerTest extends MongoTestContainerHelpe private SubmittedVariant variant3; @BeforeEach - public void setUp() throws AccessionCouldNotBeGeneratedException { + public void setUp() throws AccessionCouldNotBeGeneratedException, SQLException { repository.deleteAll(); mongoTemplate.dropCollection(DbsnpSubmittedVariantEntity.class); mongoTemplate.dropCollection(DbsnpSubmittedVariantOperationEntity.class); mongoTemplate.dropCollection(SubmittedVariantEntity.class); mongoTemplate.dropCollection(SubmittedVariantOperationEntity.class); + try (Connection conn = dataSource.getConnection(); + Statement stmt = conn.createStatement()) { + stmt.execute("DROP TABLE IF EXISTS contiguous_id_blocks"); + } + ResourceDatabasePopulator populator = new ResourceDatabasePopulator(); + populator.addScript(new ClassPathResource("test-data/contiguous_id_blocks_schema.sql")); + populator.execute(dataSource); + Long CLUSTERED_VARIANT = null; variant1 = new SubmittedVariant("ASMACC01", 1101, "PROJACC01", "CHROM1", 1234, "REF", "ALT", CLUSTERED_VARIANT); variant2 = new SubmittedVariant("ASMACC02", 1102, "PROJACC02", "CHROM2", 1234, "REF", "ALT", CLUSTERED_VARIANT); @@ -160,6 +180,8 @@ public void setUp() throws AccessionCouldNotBeGeneratedException { mockController = new SubmittedVariantsRestController(mockService, mockSubmittedVariantsBeaconService); setUpContigAliasMock(); + + } private void setUpContigAliasMock() { diff --git a/eva-accession-ws/src/test/java/uk/ac/ebi/eva/accession/ws/test/MongoTestConfiguration.java b/eva-accession-ws/src/test/java/uk/ac/ebi/eva/accession/ws/test/MongoTestConfiguration.java deleted file mode 100644 index 5a4751e52..000000000 --- a/eva-accession-ws/src/test/java/uk/ac/ebi/eva/accession/ws/test/MongoTestConfiguration.java +++ /dev/null @@ -1,105 +0,0 @@ -/* - * Copyright 2021 EMBL - European Bioinformatics Institute - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package uk.ac.ebi.eva.accession.ws.test; - -import com.mongodb.ReadPreference; -import com.mongodb.WriteConcern; -import com.mongodb.client.MongoClient; -import com.mongodb.client.MongoClients; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.domain.EntityScan; -import org.springframework.boot.test.autoconfigure.data.mongo.AutoConfigureDataMongo; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Primary; -import org.springframework.data.mongodb.MongoDatabaseFactory; -import org.springframework.data.mongodb.config.EnableMongoAuditing; -import org.springframework.data.mongodb.core.MongoTemplate; -import org.springframework.data.mongodb.core.SimpleMongoClientDatabaseFactory; -import org.springframework.data.mongodb.core.WriteResultChecking; -import org.springframework.data.mongodb.core.convert.DefaultDbRefResolver; -import org.springframework.data.mongodb.core.convert.DefaultMongoTypeMapper; -import org.springframework.data.mongodb.core.convert.MappingMongoConverter; -import org.springframework.data.mongodb.core.convert.MongoCustomConversions; -import org.springframework.data.mongodb.core.mapping.MongoMappingContext; -import org.springframework.data.mongodb.repository.config.EnableMongoRepositories; - -import java.util.Collections; - -@Configuration -@EntityScan(basePackages = {"uk.ac.ebi.eva.accession.core.persistence"}) -@EnableMongoRepositories(basePackages = { - "uk.ac.ebi.eva.accession.core.persistence", - "uk.ac.ebi.ampt2d.commons.accession.persistence.mongodb.repository"}) -@EnableMongoAuditing -@AutoConfigureDataMongo -public class MongoTestConfiguration { - - @Value("${spring.data.mongodb.uri}") - private String mongoUri; - - @Value("${spring.data.mongodb.database}") - private String database; - - @Value("${human.mongodb.database}") - private String humanDatabase; - - @Value("${mongodb.read-preference:primary}") - private String readPreference; - - @Bean - @Primary - public MongoClient mongoClient() { - return MongoClients.create(mongoUri); - } - - @Bean("primaryFactory") - @Primary - public MongoDatabaseFactory mongoDbFactory() { - return new SimpleMongoClientDatabaseFactory(mongoClient(), database); - } - - @Bean - @Primary - public MongoCustomConversions mongoCustomConversions() { - return new MongoCustomConversions(Collections.emptyList()); - } - - @Bean - @Primary - public MongoMappingContext mongoMappingContext(MongoCustomConversions conversions) { - MongoMappingContext context = new MongoMappingContext(); - context.setSimpleTypeHolder(conversions.getSimpleTypeHolder()); - context.setAutoIndexCreation(false); - return context; - } - - @Bean - @Primary - public MongoTemplate mongoTemplate(MongoMappingContext mongoMappingContext, - MongoCustomConversions conversions) { - MappingMongoConverter converter = new MappingMongoConverter( - new DefaultDbRefResolver(mongoDbFactory()), mongoMappingContext); - converter.setCustomConversions(conversions); - converter.setTypeMapper(new DefaultMongoTypeMapper(null)); - converter.afterPropertiesSet(); - MongoTemplate mongoTemplate = new MongoTemplate(mongoDbFactory(), converter); - mongoTemplate.setReadPreference(ReadPreference.valueOf(readPreference)); - mongoTemplate.setWriteConcern(WriteConcern.MAJORITY); - mongoTemplate.setWriteResultChecking(WriteResultChecking.EXCEPTION); - return mongoTemplate; - } -} diff --git a/eva-accession-ws/src/test/resources/accession-ws-test.properties b/eva-accession-ws/src/test/resources/accession-ws-test.properties index 82e3e9615..cd3c25b9a 100644 --- a/eva-accession-ws/src/test/resources/accession-ws-test.properties +++ b/eva-accession-ws/src/test/resources/accession-ws-test.properties @@ -10,6 +10,11 @@ accessioning.monotonic.test-rs.blockSize=100000 accessioning.monotonic.test-rs.blockStartValue=3000000000 accessioning.monotonic.test-rs.nextBlockInterval=1000000000 +continuous.id.blocks.datasource.driver-class-name=org.hsqldb.jdbc.JDBCDriver +continuous.id.blocks.datasource.jdbc-url=jdbc:hsqldb:mem:db;sql.syntax_pgs=true;DB_CLOSE_DELAY=-1 +continuous.id.blocks.datasource.username=SA +continuous.id.blocks.datasource.password= + # This symbolic variable is required because # this will be replaced with a default value "localhost" when run locally # (due to the filtering attribute specified in this project's POM) diff --git a/eva-accession-ws/src/test/resources/test-data/contiguous_id_blocks_schema.sql b/eva-accession-ws/src/test/resources/test-data/contiguous_id_blocks_schema.sql new file mode 100644 index 000000000..32dfe369d --- /dev/null +++ b/eva-accession-ws/src/test/resources/test-data/contiguous_id_blocks_schema.sql @@ -0,0 +1,14 @@ +CREATE SEQUENCE IF NOT EXISTS contiguous_id_blocks_seq + START WITH 1 + INCREMENT BY 50; + +CREATE TABLE IF NOT EXISTS contiguous_id_blocks ( + id bigint not NULL, + application_instance_id varchar(255) not NULL, + category_id varchar(255) not NULL, + first_value bigint not NULL, + last_committed bigint not NULL, + last_value bigint not NULL, + reserved boolean NOT NULL, + last_updated_timestamp timestamp NOT NULL +); diff --git a/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/Application.java b/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/Application.java index 41bd1013b..a0bea3cdf 100644 --- a/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/Application.java +++ b/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/Application.java @@ -15,13 +15,25 @@ */ package uk.ac.ebi.eva.remapping.source; -import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.autoconfigure.batch.BatchAutoConfiguration; +import org.springframework.boot.autoconfigure.data.mongo.MongoDataAutoConfiguration; +import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration; +import org.springframework.boot.autoconfigure.jdbc.DataSourceTransactionManagerAutoConfiguration; +import org.springframework.boot.autoconfigure.orm.jpa.HibernateJpaAutoConfiguration; import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.context.annotation.Import; +import uk.ac.ebi.eva.accession.core.configuration.InMemoryBatchConfiguration; -@EnableBatchProcessing -@SpringBootApplication +@SpringBootApplication(exclude = { + MongoDataAutoConfiguration.class, + DataSourceAutoConfiguration.class, + DataSourceTransactionManagerAutoConfiguration.class, + HibernateJpaAutoConfiguration.class, + BatchAutoConfiguration.class +}) +@Import(InMemoryBatchConfiguration.class) public class Application { public static void main(String[] args) { diff --git a/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/configuration/batch/jobs/ExportSubmittedVariantsJobConfiguration.java b/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/configuration/batch/jobs/ExportSubmittedVariantsJobConfiguration.java index c6a982d6a..626d3b2d9 100644 --- a/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/configuration/batch/jobs/ExportSubmittedVariantsJobConfiguration.java +++ b/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/configuration/batch/jobs/ExportSubmittedVariantsJobConfiguration.java @@ -18,31 +18,30 @@ import org.springframework.batch.core.Job; import org.springframework.batch.core.Step; -import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.batch.core.job.builder.JobBuilder; -import org.springframework.batch.core.launch.support.RunIdIncrementer; import org.springframework.batch.core.repository.JobRepository; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Import; -import uk.ac.ebi.eva.remapping.source.configuration.BeanNames; import uk.ac.ebi.eva.remapping.source.configuration.batch.steps.ExportSubmittedVariantsStepConfiguration; +import static uk.ac.ebi.eva.remapping.source.configuration.BeanNames.EXPORT_DBSNP_SUBMITTED_VARIANTS_STEP; +import static uk.ac.ebi.eva.remapping.source.configuration.BeanNames.EXPORT_EVA_SUBMITTED_VARIANTS_STEP; +import static uk.ac.ebi.eva.remapping.source.configuration.BeanNames.EXPORT_SUBMITTED_VARIANTS_JOB; + @Configuration -@EnableBatchProcessing @Import({ExportSubmittedVariantsStepConfiguration.class}) public class ExportSubmittedVariantsJobConfiguration { - @Bean(BeanNames.EXPORT_SUBMITTED_VARIANTS_JOB) + @Bean(EXPORT_SUBMITTED_VARIANTS_JOB) public Job accessionReleaseJob( JobRepository jobRepository, - @Autowired @Qualifier(BeanNames.EXPORT_EVA_SUBMITTED_VARIANTS_STEP) Step exportEvaSubmittedVariantsStep, - @Autowired @Qualifier(BeanNames.EXPORT_DBSNP_SUBMITTED_VARIANTS_STEP) Step exportDbsnpSubmittedVariantsStep + @Autowired @Qualifier(EXPORT_EVA_SUBMITTED_VARIANTS_STEP) Step exportEvaSubmittedVariantsStep, + @Autowired @Qualifier(EXPORT_DBSNP_SUBMITTED_VARIANTS_STEP) Step exportDbsnpSubmittedVariantsStep ) { - return new JobBuilder(BeanNames.EXPORT_SUBMITTED_VARIANTS_JOB, jobRepository) - .incrementer(new RunIdIncrementer()) + return new JobBuilder(EXPORT_SUBMITTED_VARIANTS_JOB, jobRepository) .start(exportEvaSubmittedVariantsStep) .next(exportDbsnpSubmittedVariantsStep) .build(); diff --git a/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/configuration/batch/steps/ExportSubmittedVariantsStepConfiguration.java b/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/configuration/batch/steps/ExportSubmittedVariantsStepConfiguration.java index 14af3acba..f4d956766 100644 --- a/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/configuration/batch/steps/ExportSubmittedVariantsStepConfiguration.java +++ b/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/configuration/batch/steps/ExportSubmittedVariantsStepConfiguration.java @@ -41,6 +41,8 @@ import uk.ac.ebi.eva.remapping.source.configuration.batch.policies.PoliciesConfiguration; import uk.ac.ebi.eva.remapping.source.configuration.batch.processors.SubmittedVariantsProcessorConfiguration; +import static uk.ac.ebi.eva.accession.core.configuration.InMemoryBatchConfiguration.BATCH_TRANSACTION_MANAGER; + @Configuration @Import({SubmittedVariantMongoReaderConfiguration.class, SubmittedVariantsProcessorConfiguration.class, @@ -51,7 +53,8 @@ public class ExportSubmittedVariantsStepConfiguration { @Bean(BeanNames.EXPORT_EVA_SUBMITTED_VARIANTS_STEP) public Step exportEvaSubmittedVariantsStep( - JobRepository jobRepository, PlatformTransactionManager transactionManager, + JobRepository jobRepository, + @Qualifier(BATCH_TRANSACTION_MANAGER) PlatformTransactionManager transactionManager, SimpleCompletionPolicy chunkSizeCompletionPolicy, @Autowired @Qualifier(BeanNames.EVA_SUBMITTED_VARIANT_READER) ItemReader variantReader, @Autowired @Qualifier(BeanNames.SUBMITTED_VARIANT_PROCESSOR) ItemProcessor variantProcessor, @@ -74,7 +77,8 @@ public Step exportEvaSubmittedVariantsStep( @Bean(BeanNames.EXPORT_DBSNP_SUBMITTED_VARIANTS_STEP) public Step exportDbsnpSubmittedVariantsStep( - JobRepository jobRepository, PlatformTransactionManager transactionManager, + JobRepository jobRepository, + @Qualifier(BATCH_TRANSACTION_MANAGER) PlatformTransactionManager transactionManager, SimpleCompletionPolicy chunkSizeCompletionPolicy, @Autowired @Qualifier(BeanNames.DBSNP_SUBMITTED_VARIANT_READER) ItemReader variantReader, @Autowired @Qualifier(BeanNames.SUBMITTED_VARIANT_PROCESSOR) ItemProcessor variantProcessor, diff --git a/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/parameters/InputParameters.java b/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/parameters/InputParameters.java index 4f2d34406..080d01d3b 100644 --- a/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/parameters/InputParameters.java +++ b/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/parameters/InputParameters.java @@ -35,8 +35,6 @@ public class InputParameters { private int taxonomy; - private boolean forceRestart; - private int chunkSize; public JobParameters toJobParameters() { @@ -82,14 +80,6 @@ public void setOutputFolder(String outputFolder) { this.outputFolder = outputFolder; } - public boolean isForceRestart() { - return forceRestart; - } - - public void setForceRestart(boolean forceRestart) { - this.forceRestart = forceRestart; - } - public int getChunkSize() { return chunkSize; } diff --git a/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/runner/AccessionRemappingJobLauncherCommandLineRunner.java b/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/runner/AccessionRemappingJobLauncherCommandLineRunner.java index ee20d70ae..39a7bfbf7 100644 --- a/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/runner/AccessionRemappingJobLauncherCommandLineRunner.java +++ b/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/runner/AccessionRemappingJobLauncherCommandLineRunner.java @@ -18,7 +18,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.batch.core.Job; -import org.springframework.batch.core.JobExecution; import org.springframework.batch.core.JobExecutionException; import org.springframework.batch.core.JobParameters; import org.springframework.batch.core.JobParametersInvalidException; @@ -35,10 +34,8 @@ import org.springframework.context.ApplicationEventPublisherAware; import org.springframework.stereotype.Component; import org.springframework.util.PatternMatchUtils; -import uk.ac.ebi.eva.accession.core.runner.CommandLineRunnerUtils; import uk.ac.ebi.eva.commons.batch.exception.NoJobToExecuteException; import uk.ac.ebi.eva.commons.batch.exception.NoParametersHaveBeenPassedException; -import uk.ac.ebi.eva.commons.batch.exception.NoPreviousJobExecutionException; import uk.ac.ebi.eva.commons.batch.exception.UnknownJobException; import uk.ac.ebi.eva.commons.batch.job.JobExecutionApplicationListener; import uk.ac.ebi.eva.commons.batch.job.JobStatusManager; @@ -115,18 +112,9 @@ public void run(String... args) throws JobExecutionException { JobStatusManager.checkIfJobNameHasBeenDefined(jobName); JobStatusManager.checkIfPropertiesHaveBeenProvided(jobParameters); - if (inputParameters.isForceRestart()) { - JobExecution previousJobExecution = CommandLineRunnerUtils.getLastJobExecution(jobName, jobExplorer, - jobParameters); - CommandLineRunnerUtils.markPreviousJobAsFailed(jobName, jobRepository, - previousJobExecution != null ? previousJobExecution.getJobParameters() : jobParameters); - } else { - jobParameters = CommandLineRunnerUtils.addRunIDToJobParameters(jobName, jobExplorer, jobParameters); - } - launchJob(jobParameters); - } catch (NoJobToExecuteException | NoParametersHaveBeenPassedException | NoPreviousJobExecutionException - | UnknownJobException | JobParametersInvalidException | JobExecutionAlreadyRunningException e) { + } catch (NoJobToExecuteException | NoParametersHaveBeenPassedException | UnknownJobException + | JobParametersInvalidException | JobExecutionAlreadyRunningException e) { logger.error(e.getMessage()); logger.debug("Error trace", e); abnormalExit = true; diff --git a/eva-remapping-get-source/src/main/resources/application.properties b/eva-remapping-get-source/src/main/resources/application.properties index 5425f1946..8d9ef217b 100644 --- a/eva-remapping-get-source/src/main/resources/application.properties +++ b/eva-remapping-get-source/src/main/resources/application.properties @@ -12,19 +12,10 @@ parameters.taxonomy= parameters.assemblyReportUrl= parameters.outputFolder= -parameters.forceRestart=false # this only affects the frequency of the logs parameters.chunkSize=1000 - -# job repository datasource -spring.datasource.driver-class-name=org.postgresql.Driver -spring.datasource.url= -spring.datasource.username= -spring.datasource.password= -spring.datasource.tomcat.max-active=3 - # Only to set up the database! # spring.jpa.generate-ddl=true diff --git a/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/batch/io/DbsnpSubmittedVariantMongoReaderTest.java b/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/batch/io/DbsnpSubmittedVariantMongoReaderTest.java index c397c49ab..aee09b0e0 100644 --- a/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/batch/io/DbsnpSubmittedVariantMongoReaderTest.java +++ b/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/batch/io/DbsnpSubmittedVariantMongoReaderTest.java @@ -30,9 +30,9 @@ import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.MongoConfiguration; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpSubmittedVariantEntity; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.core.utils.MongoTestDataLoader; -import uk.ac.ebi.eva.remapping.source.test.configuration.MongoTestConfiguration; import java.util.ArrayList; import java.util.Arrays; diff --git a/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/batch/io/EvaSubmittedVariantMongoReaderTest.java b/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/batch/io/EvaSubmittedVariantMongoReaderTest.java index 6925f93e2..18264e977 100644 --- a/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/batch/io/EvaSubmittedVariantMongoReaderTest.java +++ b/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/batch/io/EvaSubmittedVariantMongoReaderTest.java @@ -30,9 +30,9 @@ import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.MongoConfiguration; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.core.utils.MongoTestDataLoader; -import uk.ac.ebi.eva.remapping.source.test.configuration.MongoTestConfiguration; import java.util.ArrayList; import java.util.Collections; diff --git a/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/configuration/batch/jobs/ExportSubmittedVariantsJobConfigurationTest.java b/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/configuration/batch/jobs/ExportSubmittedVariantsJobConfigurationTest.java index 6544bb0c9..6f9212f22 100644 --- a/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/configuration/batch/jobs/ExportSubmittedVariantsJobConfigurationTest.java +++ b/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/configuration/batch/jobs/ExportSubmittedVariantsJobConfigurationTest.java @@ -30,15 +30,14 @@ import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit.jupiter.SpringExtension; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.core.utils.MongoTestDataLoader; import uk.ac.ebi.eva.commons.core.utils.FileUtils; import uk.ac.ebi.eva.remapping.source.configuration.BeanNames; import uk.ac.ebi.eva.remapping.source.parameters.InputParameters; import uk.ac.ebi.eva.remapping.source.parameters.ReportPathResolver; -import uk.ac.ebi.eva.remapping.source.test.configuration.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.remapping.source.test.configuration.BatchTestConfiguration; -import uk.ac.ebi.eva.remapping.source.test.configuration.MongoTestConfiguration; import java.io.FileInputStream; import java.io.FileNotFoundException; @@ -52,8 +51,7 @@ @ExtendWith(SpringExtension.class) @TestPropertySource("classpath:with-projects.properties") -@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class, - BatchJobRepositoryTestConfiguration.class}) +@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class}) public class ExportSubmittedVariantsJobConfigurationTest extends MongoTestContainerHelper { private static final long EXPECTED_LINES_DBSNP = 3; diff --git a/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/runner/AccessionRemappingJobLauncherCommandLineRunnerTest.java b/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/runner/AccessionRemappingJobLauncherCommandLineRunnerTest.java index 358595a01..92856ee8e 100644 --- a/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/runner/AccessionRemappingJobLauncherCommandLineRunnerTest.java +++ b/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/runner/AccessionRemappingJobLauncherCommandLineRunnerTest.java @@ -26,19 +26,17 @@ import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit.jupiter.SpringExtension; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.core.utils.MongoTestDataLoader; import uk.ac.ebi.eva.remapping.source.parameters.InputParameters; import uk.ac.ebi.eva.remapping.source.parameters.ReportPathResolver; -import uk.ac.ebi.eva.remapping.source.test.configuration.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.remapping.source.test.configuration.BatchTestConfiguration; -import uk.ac.ebi.eva.remapping.source.test.configuration.MongoTestConfiguration; import static org.junit.jupiter.api.Assertions.assertEquals; @ExtendWith(SpringExtension.class) -@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class, - BatchJobRepositoryTestConfiguration.class}) +@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class}) @TestPropertySource("classpath:application.properties") public class AccessionRemappingJobLauncherCommandLineRunnerTest extends MongoTestContainerHelper { @Autowired diff --git a/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/runner/AccessionRemappingJobLauncherCommandLineRunnerWithProjectsTest.java b/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/runner/AccessionRemappingJobLauncherCommandLineRunnerWithProjectsTest.java index 37a40d69a..ea64a4fc6 100644 --- a/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/runner/AccessionRemappingJobLauncherCommandLineRunnerWithProjectsTest.java +++ b/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/runner/AccessionRemappingJobLauncherCommandLineRunnerWithProjectsTest.java @@ -26,19 +26,17 @@ import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit.jupiter.SpringExtension; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.core.utils.MongoTestDataLoader; import uk.ac.ebi.eva.remapping.source.parameters.InputParameters; import uk.ac.ebi.eva.remapping.source.parameters.ReportPathResolver; -import uk.ac.ebi.eva.remapping.source.test.configuration.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.remapping.source.test.configuration.BatchTestConfiguration; -import uk.ac.ebi.eva.remapping.source.test.configuration.MongoTestConfiguration; import static org.junit.jupiter.api.Assertions.assertEquals; @ExtendWith(SpringExtension.class) -@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class, - BatchJobRepositoryTestConfiguration.class}) +@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class}) @TestPropertySource("classpath:with-projects.properties") public class AccessionRemappingJobLauncherCommandLineRunnerWithProjectsTest extends MongoTestContainerHelper { diff --git a/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/test/configuration/BatchJobRepositoryTestConfiguration.java b/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/test/configuration/BatchJobRepositoryTestConfiguration.java deleted file mode 100644 index 57c837cc4..000000000 --- a/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/test/configuration/BatchJobRepositoryTestConfiguration.java +++ /dev/null @@ -1,22 +0,0 @@ -package uk.ac.ebi.eva.remapping.source.test.configuration; - -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Primary; -import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseBuilder; -import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseType; - -import javax.sql.DataSource; - -@Configuration -public class BatchJobRepositoryTestConfiguration { - - @Bean - @Primary - public DataSource dataSource() { - return new EmbeddedDatabaseBuilder() - .setType(EmbeddedDatabaseType.HSQL) - .addScript("org/springframework/batch/core/schema-hsqldb.sql") - .build(); - } -} \ No newline at end of file diff --git a/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/test/configuration/BatchTestConfiguration.java b/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/test/configuration/BatchTestConfiguration.java index 9a15f97c3..701d45923 100644 --- a/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/test/configuration/BatchTestConfiguration.java +++ b/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/test/configuration/BatchTestConfiguration.java @@ -21,9 +21,9 @@ import org.springframework.batch.core.repository.JobRepository; import org.springframework.batch.test.JobLauncherTestUtils; import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Import; +import uk.ac.ebi.eva.accession.core.configuration.InMemoryBatchConfiguration; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.MongoConfiguration; import uk.ac.ebi.eva.commons.batch.job.JobExecutionApplicationListener; import uk.ac.ebi.eva.remapping.source.configuration.InputParametersConfiguration; @@ -33,8 +33,8 @@ import static uk.ac.ebi.eva.remapping.source.configuration.BeanNames.EXPORT_SUBMITTED_VARIANTS_JOB; -@EnableAutoConfiguration -@Import({MongoConfiguration.class, +@Import({InMemoryBatchConfiguration.class, + MongoConfiguration.class, InputParametersConfiguration.class, ExportSubmittedVariantsJobConfiguration.class, PoliciesConfiguration.class, diff --git a/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/test/configuration/MongoTestConfiguration.java b/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/test/configuration/MongoTestConfiguration.java deleted file mode 100644 index 1005a6f6a..000000000 --- a/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/test/configuration/MongoTestConfiguration.java +++ /dev/null @@ -1,105 +0,0 @@ -/* - * - * Copyright 2018 EMBL - European Bioinformatics Institute - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -package uk.ac.ebi.eva.remapping.source.test.configuration; - -import com.mongodb.ReadPreference; -import com.mongodb.WriteConcern; -import com.mongodb.client.MongoClient; -import com.mongodb.client.MongoClients; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.domain.EntityScan; -import org.springframework.boot.test.autoconfigure.data.mongo.AutoConfigureDataMongo; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Primary; -import org.springframework.data.mongodb.MongoDatabaseFactory; -import org.springframework.data.mongodb.config.EnableMongoAuditing; -import org.springframework.data.mongodb.core.MongoTemplate; -import org.springframework.data.mongodb.core.SimpleMongoClientDatabaseFactory; -import org.springframework.data.mongodb.core.WriteResultChecking; -import org.springframework.data.mongodb.core.convert.DefaultDbRefResolver; -import org.springframework.data.mongodb.core.convert.DefaultMongoTypeMapper; -import org.springframework.data.mongodb.core.convert.MappingMongoConverter; -import org.springframework.data.mongodb.core.convert.MongoCustomConversions; -import org.springframework.data.mongodb.core.mapping.MongoMappingContext; -import org.springframework.data.mongodb.repository.config.EnableMongoRepositories; - -import java.util.Collections; - -@Configuration -@EntityScan(basePackages = {"uk.ac.ebi.eva.accession.core.persistence"}) -@EnableMongoRepositories(basePackages = { - "uk.ac.ebi.eva.accession.core.persistence", - "uk.ac.ebi.ampt2d.commons.accession.persistence.mongodb.repository"}) -@EnableMongoAuditing -@AutoConfigureDataMongo -public class MongoTestConfiguration { - - @Value("${spring.data.mongodb.uri}") - private String mongoUri; - - @Value("${spring.data.mongodb.database}") - private String database; - - @Value("${mongodb.read-preference:primary}") - private String readPreference; - - @Bean - @Primary - public MongoClient mongoClient() { - return MongoClients.create(mongoUri); - } - - @Bean("primaryFactory") - @Primary - public MongoDatabaseFactory mongoDbFactory() { - return new SimpleMongoClientDatabaseFactory(mongoClient(), database); - } - - @Bean - @Primary - public MongoCustomConversions mongoCustomConversions() { - return new MongoCustomConversions(Collections.emptyList()); - } - - @Bean - @Primary - public MongoMappingContext mongoMappingContext(MongoCustomConversions conversions) { - MongoMappingContext context = new MongoMappingContext(); - context.setSimpleTypeHolder(conversions.getSimpleTypeHolder()); - context.setAutoIndexCreation(false); - return context; - } - - @Bean - @Primary - public MongoTemplate mongoTemplate(MongoMappingContext mongoMappingContext, - MongoCustomConversions conversions) { - MappingMongoConverter converter = new MappingMongoConverter( - new DefaultDbRefResolver(mongoDbFactory()), mongoMappingContext); - converter.setCustomConversions(conversions); - converter.setTypeMapper(new DefaultMongoTypeMapper(null)); - converter.afterPropertiesSet(); - MongoTemplate mongoTemplate = new MongoTemplate(mongoDbFactory(), converter); - mongoTemplate.setReadPreference(ReadPreference.valueOf(readPreference)); - mongoTemplate.setWriteConcern(WriteConcern.MAJORITY); - mongoTemplate.setWriteResultChecking(WriteResultChecking.EXCEPTION); - return mongoTemplate; - } - -} diff --git a/eva-remapping-get-source/src/test/resources/application.properties b/eva-remapping-get-source/src/test/resources/application.properties index 941cf32a5..cb9e7fe5d 100644 --- a/eva-remapping-get-source/src/test/resources/application.properties +++ b/eva-remapping-get-source/src/test/resources/application.properties @@ -13,7 +13,6 @@ parameters.assemblyAccession=GCA_000409795.2 parameters.fasta=src/test/resources/input-files/fasta/GCF_000409795.2.test.fna parameters.assemblyReportUrl=file:src/test/resources/input-files/assembly-report/GCF_000409795.2_Chlorocebus_sabeus_1.1_assembly_report.txt parameters.outputFolder=/tmp/ -parameters.forceRestart=false parameters.chunkSize=1000 spring.data.mongodb.database=test-db diff --git a/eva-remapping-get-source/src/test/resources/with-projects.properties b/eva-remapping-get-source/src/test/resources/with-projects.properties index d0c6bb68d..9c008b83a 100644 --- a/eva-remapping-get-source/src/test/resources/with-projects.properties +++ b/eva-remapping-get-source/src/test/resources/with-projects.properties @@ -15,7 +15,6 @@ parameters.taxonomy=60711 parameters.fasta=src/test/resources/input-files/fasta/GCF_000409795.2.test.fna parameters.assemblyReportUrl=file:src/test/resources/input-files/assembly-report/GCF_000409795.2_Chlorocebus_sabeus_1.1_assembly_report.txt parameters.outputFolder=/tmp/ -parameters.forceRestart=false parameters.chunkSize=1000 spring.data.mongodb.database=test-db diff --git a/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/Application.java b/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/Application.java index a8e45d5b4..f6c27e636 100644 --- a/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/Application.java +++ b/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/Application.java @@ -15,13 +15,25 @@ */ package uk.ac.ebi.eva.remapping.ingest; -import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.autoconfigure.batch.BatchAutoConfiguration; +import org.springframework.boot.autoconfigure.data.mongo.MongoDataAutoConfiguration; +import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration; +import org.springframework.boot.autoconfigure.jdbc.DataSourceTransactionManagerAutoConfiguration; +import org.springframework.boot.autoconfigure.orm.jpa.HibernateJpaAutoConfiguration; import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.context.annotation.Import; +import uk.ac.ebi.eva.accession.core.configuration.InMemoryBatchConfiguration; -@EnableBatchProcessing -@SpringBootApplication +@SpringBootApplication(exclude = { + MongoDataAutoConfiguration.class, + DataSourceAutoConfiguration.class, + DataSourceTransactionManagerAutoConfiguration.class, + HibernateJpaAutoConfiguration.class, + BatchAutoConfiguration.class +}) +@Import(InMemoryBatchConfiguration.class) public class Application { public static void main(String[] args) { diff --git a/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/jobs/IngestRemappedVariantsFromVcfJobConfiguration.java b/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/jobs/IngestRemappedVariantsFromVcfJobConfiguration.java index d7cee50c2..c0169881f 100644 --- a/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/jobs/IngestRemappedVariantsFromVcfJobConfiguration.java +++ b/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/jobs/IngestRemappedVariantsFromVcfJobConfiguration.java @@ -17,9 +17,7 @@ import org.springframework.batch.core.Job; import org.springframework.batch.core.Step; -import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.batch.core.job.builder.JobBuilder; -import org.springframework.batch.core.launch.support.RunIdIncrementer; import org.springframework.batch.core.repository.JobRepository; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; @@ -30,7 +28,6 @@ import static uk.ac.ebi.eva.remapping.ingest.configuration.BeanNames.STORE_REMAPPING_METADATA_STEP; @Configuration -@EnableBatchProcessing public class IngestRemappedVariantsFromVcfJobConfiguration { @Bean(INGEST_REMAPPED_VARIANTS_FROM_VCF_JOB) @@ -39,7 +36,6 @@ public Job ingestRemappedVariantsFromVcfJob( @Qualifier(STORE_REMAPPING_METADATA_STEP) Step storeRemappingMetadataStep, JobRepository jobRepository) { return new JobBuilder(INGEST_REMAPPED_VARIANTS_FROM_VCF_JOB, jobRepository) - .incrementer(new RunIdIncrementer()) .start(storeRemappingMetadataStep) .next(ingestRemappedVariantsStep) .build(); diff --git a/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/steps/IngestRemappedFromVcfStepConfiguration.java b/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/steps/IngestRemappedFromVcfStepConfiguration.java index e123343ca..98962018d 100644 --- a/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/steps/IngestRemappedFromVcfStepConfiguration.java +++ b/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/steps/IngestRemappedFromVcfStepConfiguration.java @@ -18,7 +18,6 @@ import org.springframework.batch.core.Step; import org.springframework.batch.core.StepExecutionListener; -import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.batch.core.repository.JobRepository; import org.springframework.batch.core.step.builder.StepBuilder; import org.springframework.batch.core.step.tasklet.TaskletStep; @@ -35,13 +34,13 @@ import uk.ac.ebi.eva.commons.core.models.pipeline.Variant; import uk.ac.ebi.eva.remapping.ingest.configuration.BeanNames; +import static uk.ac.ebi.eva.accession.core.configuration.InMemoryBatchConfiguration.BATCH_TRANSACTION_MANAGER; import static uk.ac.ebi.eva.remapping.ingest.configuration.BeanNames.COMPOSITE_VARIANT_PROCESSOR; import static uk.ac.ebi.eva.remapping.ingest.configuration.BeanNames.PROGRESS_LISTENER; import static uk.ac.ebi.eva.remapping.ingest.configuration.BeanNames.REMAPPED_SUBMITTED_VARIANTS_WRITER; import static uk.ac.ebi.eva.remapping.ingest.configuration.BeanNames.VCF_READER; @Configuration -@EnableBatchProcessing public class IngestRemappedFromVcfStepConfiguration { @Bean(BeanNames.INGEST_REMAPPED_VARIANTS_FROM_VCF_STEP) @@ -50,7 +49,8 @@ public Step ingestRemappedFromVcf( @Qualifier(COMPOSITE_VARIANT_PROCESSOR) ItemProcessor processor, @Qualifier(REMAPPED_SUBMITTED_VARIANTS_WRITER) ItemWriter submittedVariantWriter, @Qualifier(PROGRESS_LISTENER) StepExecutionListener progressListener, - JobRepository jobRepository, PlatformTransactionManager transactionManager, + JobRepository jobRepository, + @Qualifier(BATCH_TRANSACTION_MANAGER) PlatformTransactionManager transactionManager, SimpleCompletionPolicy chunkSizeCompletionPolicy) { TaskletStep step = new StepBuilder(BeanNames.INGEST_REMAPPED_VARIANTS_FROM_VCF_STEP, jobRepository) .chunk(chunkSizeCompletionPolicy, transactionManager) diff --git a/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/steps/StoreRemappingMetadataStepConfiguration.java b/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/steps/StoreRemappingMetadataStepConfiguration.java index 6d2c6803b..270b3d254 100644 --- a/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/steps/StoreRemappingMetadataStepConfiguration.java +++ b/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/steps/StoreRemappingMetadataStepConfiguration.java @@ -17,10 +17,10 @@ package uk.ac.ebi.eva.remapping.ingest.configuration.batch.steps; import org.springframework.batch.core.Step; -import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.batch.core.repository.JobRepository; import org.springframework.batch.core.step.builder.StepBuilder; import org.springframework.batch.core.step.tasklet.TaskletStep; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.data.mongodb.core.MongoTemplate; @@ -28,12 +28,10 @@ import uk.ac.ebi.eva.remapping.ingest.batch.tasklets.RemappingMetadata; import uk.ac.ebi.eva.remapping.ingest.batch.tasklets.StoreRemappingMetadataTasklet; -import java.io.IOException; - +import static uk.ac.ebi.eva.accession.core.configuration.InMemoryBatchConfiguration.BATCH_TRANSACTION_MANAGER; import static uk.ac.ebi.eva.remapping.ingest.configuration.BeanNames.STORE_REMAPPING_METADATA_STEP; @Configuration -@EnableBatchProcessing public class StoreRemappingMetadataStepConfiguration { private final MongoTemplate mongoTemplate; @@ -46,7 +44,8 @@ public StoreRemappingMetadataStepConfiguration(MongoTemplate mongoTemplate, Rema } @Bean(STORE_REMAPPING_METADATA_STEP) - public Step buildReportStep(JobRepository jobRepository, PlatformTransactionManager transactionManager) throws IOException { + public Step buildReportStep(JobRepository jobRepository, + @Qualifier(BATCH_TRANSACTION_MANAGER) PlatformTransactionManager transactionManager) { StoreRemappingMetadataTasklet tasklet = new StoreRemappingMetadataTasklet(mongoTemplate, remappingMetadata); TaskletStep step = new StepBuilder(STORE_REMAPPING_METADATA_STEP, jobRepository) .tasklet(tasklet, transactionManager) diff --git a/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/parameters/InputParameters.java b/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/parameters/InputParameters.java index 572b7333b..f0377628e 100644 --- a/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/parameters/InputParameters.java +++ b/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/parameters/InputParameters.java @@ -34,8 +34,6 @@ public class InputParameters { private int chunkSize; - private boolean forceRestart; - public String getVcf() { return vcf; } @@ -92,14 +90,6 @@ public void setChunkSize(int chunkSize) { this.chunkSize = chunkSize; } - public boolean isForceRestart() { - return forceRestart; - } - - public void setForceRestart(boolean forceRestart) { - this.forceRestart = forceRestart; - } - public JobParameters toJobParameters() { return new JobParametersBuilder() .addString("remappedFrom", remappedFrom) diff --git a/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/runner/IngestRemappedCommandLineRunner.java b/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/runner/IngestRemappedCommandLineRunner.java index 021002fd9..e45354396 100644 --- a/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/runner/IngestRemappedCommandLineRunner.java +++ b/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/runner/IngestRemappedCommandLineRunner.java @@ -20,7 +20,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.batch.core.Job; -import org.springframework.batch.core.JobExecution; import org.springframework.batch.core.JobExecutionException; import org.springframework.batch.core.JobParameters; import org.springframework.batch.core.JobParametersInvalidException; @@ -37,10 +36,8 @@ import org.springframework.context.ApplicationEventPublisherAware; import org.springframework.stereotype.Component; import org.springframework.util.PatternMatchUtils; -import uk.ac.ebi.eva.accession.core.runner.CommandLineRunnerUtils; import uk.ac.ebi.eva.commons.batch.exception.NoJobToExecuteException; import uk.ac.ebi.eva.commons.batch.exception.NoParametersHaveBeenPassedException; -import uk.ac.ebi.eva.commons.batch.exception.NoPreviousJobExecutionException; import uk.ac.ebi.eva.commons.batch.exception.UnknownJobException; import uk.ac.ebi.eva.commons.batch.job.JobExecutionApplicationListener; import uk.ac.ebi.eva.commons.batch.job.JobStatusManager; @@ -114,17 +111,10 @@ public void run(String... args) throws JobExecutionException { JobStatusManager.checkIfJobNameHasBeenDefined(jobName); JobStatusManager.checkIfPropertiesHaveBeenProvided(jobParameters); - if (inputParameters.isForceRestart()) { - JobExecution previousJobExecution = CommandLineRunnerUtils.getLastJobExecution(jobName, jobExplorer, - jobParameters); - CommandLineRunnerUtils.markPreviousJobAsFailed(jobName, jobRepository, - previousJobExecution != null ? previousJobExecution.getJobParameters() : jobParameters); - } else { - jobParameters = CommandLineRunnerUtils.addRunIDToJobParameters(jobName, jobExplorer, jobParameters); - } + launchJob(jobParameters); - } catch (NoJobToExecuteException | NoParametersHaveBeenPassedException | NoPreviousJobExecutionException - | UnknownJobException | JobParametersInvalidException | JobExecutionAlreadyRunningException e) { + } catch (NoJobToExecuteException | NoParametersHaveBeenPassedException | UnknownJobException + | JobParametersInvalidException | JobExecutionAlreadyRunningException e) { logger.error(e.getMessage()); logger.debug("Error trace", e); abnormalExit = true; diff --git a/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/batch/io/RemappedSubmittedVariantsWriterTest.java b/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/batch/io/RemappedSubmittedVariantsWriterTest.java index 316e09f93..73a62faf3 100644 --- a/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/batch/io/RemappedSubmittedVariantsWriterTest.java +++ b/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/batch/io/RemappedSubmittedVariantsWriterTest.java @@ -34,10 +34,10 @@ import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantOperationEntity; import uk.ac.ebi.eva.accession.core.summary.SubmittedVariantSummaryFunction; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.remapping.ingest.batch.listeners.RemappingIngestCounts; import uk.ac.ebi.eva.remapping.ingest.test.configuration.BatchTestConfiguration; -import uk.ac.ebi.eva.remapping.ingest.test.configuration.MongoTestConfiguration; import java.time.LocalDateTime; import java.util.Arrays; diff --git a/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/jobs/IngestRemappedVariantsFromVcfJobConfigurationTest.java b/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/jobs/IngestRemappedVariantsFromVcfJobConfigurationTest.java index 232ec7218..656c20c87 100644 --- a/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/jobs/IngestRemappedVariantsFromVcfJobConfigurationTest.java +++ b/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/jobs/IngestRemappedVariantsFromVcfJobConfigurationTest.java @@ -34,12 +34,11 @@ import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.core.utils.MongoTestDataLoader; import uk.ac.ebi.eva.remapping.ingest.batch.tasklets.RemappingMetadata; -import uk.ac.ebi.eva.remapping.ingest.test.configuration.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.remapping.ingest.test.configuration.BatchTestConfiguration; -import uk.ac.ebi.eva.remapping.ingest.test.configuration.MongoTestConfiguration; import java.util.Arrays; import java.util.Collection; @@ -53,8 +52,7 @@ import static uk.ac.ebi.eva.remapping.ingest.test.configuration.BatchTestConfiguration.JOB_INGEST_REMAPPED_VARIANTS_FROM_VCF_JOB; @ExtendWith(SpringExtension.class) -@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class, - BatchJobRepositoryTestConfiguration.class}) +@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class}) @TestPropertySource("classpath:ingest-remapped-variants.properties") public class IngestRemappedVariantsFromVcfJobConfigurationTest extends MongoTestContainerHelper { diff --git a/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/steps/IngestRemappedFromVcfStepConfigurationTest.java b/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/steps/IngestRemappedFromVcfStepConfigurationTest.java index 931ba1175..7fbddaf0c 100644 --- a/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/steps/IngestRemappedFromVcfStepConfigurationTest.java +++ b/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/steps/IngestRemappedFromVcfStepConfigurationTest.java @@ -33,20 +33,18 @@ import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.core.utils.MongoTestDataLoader; import uk.ac.ebi.eva.remapping.ingest.configuration.BeanNames; -import uk.ac.ebi.eva.remapping.ingest.test.configuration.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.remapping.ingest.test.configuration.BatchTestConfiguration; -import uk.ac.ebi.eva.remapping.ingest.test.configuration.MongoTestConfiguration; import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; @ExtendWith(SpringExtension.class) -@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class, - BatchJobRepositoryTestConfiguration.class}) +@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class}) @TestPropertySource("classpath:ingest-remapped-variants.properties") public class IngestRemappedFromVcfStepConfigurationTest extends MongoTestContainerHelper { diff --git a/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/steps/IngestRemappedFromVcfStepDbsnpConfigurationTest.java b/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/steps/IngestRemappedFromVcfStepDbsnpConfigurationTest.java index 377c3ee6d..aa2e218d9 100644 --- a/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/steps/IngestRemappedFromVcfStepDbsnpConfigurationTest.java +++ b/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/steps/IngestRemappedFromVcfStepDbsnpConfigurationTest.java @@ -33,20 +33,18 @@ import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpSubmittedVariantEntity; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.core.utils.MongoTestDataLoader; import uk.ac.ebi.eva.remapping.ingest.configuration.BeanNames; -import uk.ac.ebi.eva.remapping.ingest.test.configuration.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.remapping.ingest.test.configuration.BatchTestConfiguration; -import uk.ac.ebi.eva.remapping.ingest.test.configuration.MongoTestConfiguration; import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; @ExtendWith(SpringExtension.class) -@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class, - BatchJobRepositoryTestConfiguration.class}) +@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class}) @TestPropertySource("classpath:ingest-remapped-variants-dbsnp.properties") public class IngestRemappedFromVcfStepDbsnpConfigurationTest extends MongoTestContainerHelper { diff --git a/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/test/configuration/BatchJobRepositoryTestConfiguration.java b/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/test/configuration/BatchJobRepositoryTestConfiguration.java deleted file mode 100644 index cfa3352cb..000000000 --- a/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/test/configuration/BatchJobRepositoryTestConfiguration.java +++ /dev/null @@ -1,22 +0,0 @@ -package uk.ac.ebi.eva.remapping.ingest.test.configuration; - -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Primary; -import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseBuilder; -import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseType; - -import javax.sql.DataSource; - -@Configuration -public class BatchJobRepositoryTestConfiguration { - - @Bean - @Primary - public DataSource dataSource() { - return new EmbeddedDatabaseBuilder() - .setType(EmbeddedDatabaseType.HSQL) - .addScript("org/springframework/batch/core/schema-hsqldb.sql") - .build(); - } -} \ No newline at end of file diff --git a/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/test/configuration/BatchTestConfiguration.java b/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/test/configuration/BatchTestConfiguration.java index a7065192a..97b49f4d2 100644 --- a/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/test/configuration/BatchTestConfiguration.java +++ b/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/test/configuration/BatchTestConfiguration.java @@ -22,10 +22,10 @@ import org.springframework.batch.test.JobLauncherTestUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Import; import org.springframework.core.io.ResourceLoader; +import uk.ac.ebi.eva.accession.core.configuration.InMemoryBatchConfiguration; import uk.ac.ebi.eva.commons.batch.job.JobExecutionApplicationListener; import uk.ac.ebi.eva.remapping.ingest.configuration.RemappingMetadataConfiguration; import uk.ac.ebi.eva.remapping.ingest.configuration.batch.io.IngestRemappedSubmittedVariantsWriterConfiguration; @@ -39,8 +39,8 @@ import static uk.ac.ebi.eva.remapping.ingest.configuration.BeanNames.INGEST_REMAPPED_VARIANTS_FROM_VCF_JOB; -@EnableAutoConfiguration -@Import({IngestRemappedFromVcfStepConfiguration.class, +@Import({InMemoryBatchConfiguration.class, + IngestRemappedFromVcfStepConfiguration.class, StoreRemappingMetadataStepConfiguration.class, IngestRemappedVariantsFromVcfJobConfiguration.class, VcfReaderConfiguration.class, diff --git a/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/test/configuration/MongoTestConfiguration.java b/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/test/configuration/MongoTestConfiguration.java deleted file mode 100644 index 2cab74199..000000000 --- a/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/test/configuration/MongoTestConfiguration.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Copyright 2021 EMBL - European Bioinformatics Institute - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package uk.ac.ebi.eva.remapping.ingest.test.configuration; - -import com.mongodb.ReadPreference; -import com.mongodb.WriteConcern; -import com.mongodb.client.MongoClient; -import com.mongodb.client.MongoClients; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.domain.EntityScan; -import org.springframework.boot.test.autoconfigure.data.mongo.AutoConfigureDataMongo; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Primary; -import org.springframework.data.mongodb.MongoDatabaseFactory; -import org.springframework.data.mongodb.config.EnableMongoAuditing; -import org.springframework.data.mongodb.core.MongoTemplate; -import org.springframework.data.mongodb.core.SimpleMongoClientDatabaseFactory; -import org.springframework.data.mongodb.core.WriteResultChecking; -import org.springframework.data.mongodb.core.convert.DefaultDbRefResolver; -import org.springframework.data.mongodb.core.convert.DefaultMongoTypeMapper; -import org.springframework.data.mongodb.core.convert.MappingMongoConverter; -import org.springframework.data.mongodb.core.convert.MongoCustomConversions; -import org.springframework.data.mongodb.core.mapping.MongoMappingContext; -import org.springframework.data.mongodb.repository.config.EnableMongoRepositories; - -import java.util.Collections; - -@Configuration -@EntityScan(basePackages = {"uk.ac.ebi.eva.accession.core.persistence"}) -@EnableMongoRepositories(basePackages = { - "uk.ac.ebi.eva.accession.core.persistence", - "uk.ac.ebi.ampt2d.commons.accession.persistence.mongodb.repository"}) -@EnableMongoAuditing -@AutoConfigureDataMongo -public class MongoTestConfiguration { - - @Value("${spring.data.mongodb.uri}") - private String mongoUri; - - @Value("${spring.data.mongodb.database}") - private String database; - - @Value("${mongodb.read-preference:primary}") - private String readPreference; - - @Bean - @Primary - public MongoClient mongoClient() { - return MongoClients.create(mongoUri); - } - - @Bean("primaryFactory") - @Primary - public MongoDatabaseFactory mongoDbFactory() { - return new SimpleMongoClientDatabaseFactory(mongoClient(), database); - } - - @Bean - @Primary - public MongoCustomConversions mongoCustomConversions() { - return new MongoCustomConversions(Collections.emptyList()); - } - - @Bean - @Primary - public MongoMappingContext mongoMappingContext(MongoCustomConversions conversions) { - MongoMappingContext context = new MongoMappingContext(); - context.setSimpleTypeHolder(conversions.getSimpleTypeHolder()); - context.setAutoIndexCreation(false); - return context; - } - - @Bean - @Primary - public MongoTemplate mongoTemplate(MongoMappingContext mongoMappingContext, - MongoCustomConversions conversions) { - MappingMongoConverter converter = new MappingMongoConverter( - new DefaultDbRefResolver(mongoDbFactory()), mongoMappingContext); - converter.setCustomConversions(conversions); - converter.setTypeMapper(new DefaultMongoTypeMapper(null)); - converter.afterPropertiesSet(); - MongoTemplate mongoTemplate = new MongoTemplate(mongoDbFactory(), converter); - mongoTemplate.setReadPreference(ReadPreference.valueOf(readPreference)); - mongoTemplate.setWriteConcern(WriteConcern.MAJORITY); - mongoTemplate.setWriteResultChecking(WriteResultChecking.EXCEPTION); - return mongoTemplate; - } -} diff --git a/pom.xml b/pom.xml index c0f2ee9ea..918ff0ee5 100644 --- a/pom.xml +++ b/pom.xml @@ -153,17 +153,6 @@ test - - org.postgresql - postgresql - - - - org.hsqldb - hsqldb - test - - org.springframework.boot spring-boot-testcontainers From 6992eca9960e9850dd85eb73627f5ba82f260633 Mon Sep 17 00:00:00 2001 From: nitin-ebi <79518737+nitin-ebi@users.noreply.github.com> Date: Mon, 8 Jun 2026 09:43:23 +0100 Subject: [PATCH 05/10] EVA-4113 Integration test updates (#503) --- .../configuration/RunnerConfiguration.java | 52 +++++++++++++++++++ .../pipeline/parameters/InputParameters.java | 12 +++-- 2 files changed, 60 insertions(+), 4 deletions(-) create mode 100644 eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/RunnerConfiguration.java diff --git a/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/RunnerConfiguration.java b/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/RunnerConfiguration.java new file mode 100644 index 000000000..321e73f15 --- /dev/null +++ b/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/RunnerConfiguration.java @@ -0,0 +1,52 @@ +/* + * Copyright 2018 EMBL - European Bioinformatics Institute + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package uk.ac.ebi.eva.accession.deprecate.configuration; + +import org.springframework.batch.core.ExitStatus; +import org.springframework.batch.core.Job; +import org.springframework.batch.core.JobExecution; +import org.springframework.batch.core.JobParametersBuilder; +import org.springframework.batch.core.launch.JobLauncher; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.CommandLineRunner; +import org.springframework.context.ApplicationContext; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import uk.ac.ebi.eva.commons.batch.job.JobExecutionApplicationListener; + +@Configuration +public class RunnerConfiguration { + + @Bean + public JobExecutionApplicationListener jobExecutionApplicationListener() { + return new JobExecutionApplicationListener(); + } + + @Bean + public CommandLineRunner runJob(JobLauncher jobLauncher, ApplicationContext context, + @Value("${spring.batch.job.names}") String jobName) { + return args -> { + Job job = context.getBean(jobName, Job.class); + JobExecution execution = jobLauncher.run(job, new JobParametersBuilder() + .toJobParameters()); + + if (!execution.getExitStatus().equals(ExitStatus.COMPLETED)) { + throw new RuntimeException("Job failed with status: " + execution.getExitStatus()); + } + }; + } +} \ No newline at end of file diff --git a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/parameters/InputParameters.java b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/parameters/InputParameters.java index 1f6e017e0..beca7cb77 100644 --- a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/parameters/InputParameters.java +++ b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/parameters/InputParameters.java @@ -47,18 +47,22 @@ public class InputParameters { private String duplicateSSAccFile; public JobParameters toJobParameters() { - return new JobParametersBuilder() + JobParametersBuilder builder = new JobParametersBuilder() .addString("vcf", vcf) .addString("vcfAggregation", vcfAggregation.toString()) - .addString("aggregatedMappingFile", aggregatedMappingFile) .addString("fasta", fasta) .addString("outputVcf", outputVcf) .addLong("taxonomyAccession", (long) taxonomyAccession) .addString("assemblyAccession", assemblyAccession) .addString("projectAccession", projectAccession) .addLong("chunkSize", (long) chunkSize) - .addString("contigNaming", contigNaming.toString()) - .toJobParameters(); + .addString("contigNaming", contigNaming.toString()); + + if (aggregatedMappingFile != null) { + builder.addString("aggregatedMappingFile", aggregatedMappingFile); + } + + return builder.toJobParameters(); } public String getVcf() { From 7bd1fbebe4caeb9b7b5134d89f871113fd996a8e Mon Sep 17 00:00:00 2001 From: nkumar2 Date: Mon, 18 May 2026 13:51:48 +0100 Subject: [PATCH 06/10] add java21 branch for testing --- .github/workflows/run-tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index e82393b3d..6bd664e22 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -2,10 +2,10 @@ name: Run tests on: push: - branches: [ master ] + branches: [ master, java21 ] pull_request: # Temporarily run tests on the feature branch until merged to master - branches: [ master, feature/ws-contig-alias-integration ] + branches: [ master, java21, feature/ws-contig-alias-integration ] jobs: build: From a100195abd5df8396ad99fec9d27e49f3b65fb95 Mon Sep 17 00:00:00 2001 From: nitin-ebi <79518737+nitin-ebi@users.noreply.github.com> Date: Mon, 1 Jun 2026 19:51:31 +0100 Subject: [PATCH 07/10] Upgrade project - java 21 - spring 3.4 (#501) --- .github/workflows/run-tests.yml | 42 +-- .gitlab-ci.yml | 2 +- eva-accession-clustering/pom.xml | 54 +++- .../batch/io/BackPropagatedRSWriter.java | 26 +- .../clustering/batch/io/ClusteringWriter.java | 127 ++++---- .../batch/io/ListOfListItemWriter.java | 7 +- .../clustering/batch/io/RSMergeWriter.java | 134 ++++---- .../clustering/batch/io/RSSplitWriter.java | 68 ++-- .../clustering/batch/io/SSSplitWriter.java | 85 +++-- ...getSSReaderForSplitOrMergedBackPropRS.java | 15 +- .../batch/io/qc/DuplicateRSAccQCWriter.java | 7 +- .../batch/io/qc/ExtraneousRSReporter.java | 22 +- .../batch/io/qc/MissingCveReporter.java | 12 +- .../io/qc/PendingMergeSplitReporter.java | 37 +-- .../clustering/batch/io/qc/RSReader.java | 11 +- .../clustering/batch/io/qc/SSReader.java | 12 +- ...iantToSubmittedVariantEntityProcessor.java | 5 +- .../batch/RunnerConfiguration.java | 12 - ...AndSplitCandidatesReaderConfiguration.java | 30 +- .../jobs/BackPropagateRSJobConfiguration.java | 16 +- ...erUnclusteredVariantsJobConfiguration.java | 23 +- .../ClusteringFromMongoJobConfiguration.java | 67 ++-- ...emappedVariantsWithRSJobConfiguration.java | 13 +- .../RSAccessionRecoveryJobConfiguration.java | 7 +- .../jobs/StudyClusteringJobConfiguration.java | 17 +- .../jobs/qc/ClusteringQCJobConfiguration.java | 32 +- .../qc/DuplicateRSAccQCJobConfiguration.java | 7 +- ...ewClusteredVariantsQCJobConfiguration.java | 26 +- ...AccessioningShutdownStepConfiguration.java | 10 +- .../ClusteringFromMongoStepConfiguration.java | 128 ++++---- .../RSAccessionRecoveryStepConfiguration.java | 11 +- .../qc/DuplicateRSAccQCStepConfiguration.java | 10 +- .../metric/ClusteringMetricCompute.java | 17 +- .../parameters/InputParameters.java | 3 +- .../runner/ClusteringCommandLineRunner.java | 19 +- .../batch/io/BackPropagatedRSWriterTest.java | 62 ++-- .../ClusteredVariantSplittingPolicyTest.java | 10 +- .../io/ClusteringMongoReaderRetryTest.java | 20 +- .../batch/io/ClusteringMongoReaderTest.java | 54 ++-- .../RSMergeAndSplitCandidatesReaderTest.java | 144 ++++----- .../batch/io/RSMergeWriterTest.java | 204 ++++++------ .../batch/io/RSSplitWriterTest.java | 165 ++++------ .../batch/io/SSSplitWriterTest.java | 106 +++---- .../io/StudyClusteringMongoReaderTest.java | 54 ++-- .../ClusteringAssertions.java | 2 +- .../IssueAccessionClusteringWriterTest.java | 125 ++++---- .../MergeAccessionClusteringWriterTest.java | 145 ++++----- .../RemappedVariantsClusteringWriterTest.java | 233 +++++++------- .../ReuseAccessionClusteringWriterTest.java | 71 ++--- ...ToSubmittedVariantEntityProcessorTest.java | 10 +- ...ClusteringVariantJobConfigurationTest.java | 92 +++--- .../batch/jobs/RSAccessionRecoveryTest.java | 35 +- ...DuplicateRSAccQCStepConfigurationTest.java | 82 ++--- ...usteredVariantsQCJobConfigurationTest.java | 55 ++-- ...lusteringVariantStepConfigurationTest.java | 66 ++-- .../ClusteringCommandLineRunnerTest.java | 143 ++++----- .../clustering/test/VariantAssertions.java | 2 +- .../BatchJobRepositoryTestConfiguration.java | 22 ++ .../configuration/BatchTestConfiguration.java | 117 +++---- .../configuration/MongoTestConfiguration.java | 72 ++++- .../RSAccessionRecoveryTestConfiguration.java | 46 +-- .../test/rule/FixSpringMongoDbRule.java | 40 --- .../resources/backpropagation-test.properties | 3 - .../clustering-issuance-test.properties | 3 - .../clustering-pipeline-test.properties | 4 +- .../resources/clustering-qc-test.properties | 3 - .../clustering-writer-test.properties | 3 - .../duplicate-rs-acc-qc-test.properties | 3 - .../resources/merge-split-test.properties | 3 - .../rs-accession-recovery.properties | 10 +- eva-accession-core/pom.xml | 20 +- .../io/ClusteredVariantDeprecationWriter.java | 50 +-- .../DbsnpClusteredVariantOperationWriter.java | 12 +- .../batch/io/DbsnpClusteredVariantWriter.java | 15 +- .../io/SubmittedVariantDeprecationWriter.java | 52 +-- .../listeners/GenericProgressListener.java | 9 +- .../policies/IllegalStartSkipPolicy.java | 2 +- .../configuration/MongoClientCreator.java | 37 ++- .../human/HumanMongoConfiguration.java | 29 +- .../nonhuman/MongoConfiguration.java | 53 ++-- ...ittedVariantAccessioningConfiguration.java | 18 +- .../core/model/ClusteredVariant.java | 4 +- .../DbsnpSubmittedVariantInactiveEntity.java | 1 - .../core/runner/CommandLineRunnerUtils.java | 31 +- .../DbsnpSubmittedVariantInactiveService.java | 3 +- .../eva/SubmittedVariantInactiveService.java | 3 +- .../core/EVAObjectModelUtilsTest.java | 9 +- .../batch/io/AccessionedVcfFactoryTest.java | 15 +- ...ClusteredVariantDeprecationWriterTest.java | 83 ++--- .../io/DbsnpClusteredVariantWriterTest.java | 65 ++-- .../batch/io/FastaSequenceReaderTest.java | 68 ++-- ...SubmittedVariantDeprecationWriterTest.java | 88 ++---- .../policies/IllegalStartSkipPolicyTest.java | 10 +- .../core/contig/ContigMappingTest.java | 14 +- ...ClusteredVariantEntityIntegrationTest.java | 41 +-- ...leaseRecordSubmittedVariantEntityTest.java | 4 +- ...SubmittedVariantEntityIntegrationTest.java | 46 +-- .../model/eva/SubmittedVariantEntityTest.java | 25 +- .../core/mongoTemplate/MongoTemplateTest.java | 23 +- ...eredVariantAccessioningRepositoryTest.java | 63 ++-- ...ttedVariantAccessioningRepositoryTest.java | 81 +++-- ...usteredVariantAccessioningServiceTest.java | 60 ++-- ...usteredVariantAccessioningServiceTest.java | 49 ++- ...bmittedVariantAccessioningServiceTest.java | 235 +++++++------- ...riantMonotonicAccessioningServiceTest.java | 37 ++- .../ClusteredVariantSummaryFunctionTest.java | 10 +- .../SubmittedVariantSummaryFunctionTest.java | 10 +- .../configuration/DbsnpTestDataSource.java | 1 - .../configuration/JPATestConfiguration.java | 41 +++ .../human/MongoHumanTestConfiguration.java | 69 +++- .../nonhuman/MongoTestConfiguration.java | 70 ++++ .../core/test/rule/FixSpringMongoDbRule.java | 41 --- .../core/utils/MongoTestContainerHelper.java | 24 ++ .../core/utils/MongoTestDataLoader.java | 106 +++++++ .../utils/PipelineTemporaryFolderUtil.java | 107 +++++++ .../properties/rs-accession-test.properties | 3 - .../properties/ss-accession-test.properties | 7 - .../properties/ss-deprecation-test.properties | 7 - .../properties/test-model.properties | 3 - .../test-variants-writer.properties | 4 - eva-accession-deprecate/pom.xml | 30 +- ...ySubmittedVariantsReaderConfiguration.java | 3 +- ...tudySubmittedVariantsJobConfiguration.java | 26 +- ...ittedVariantsFromFileJobConfiguration.java | 19 +- .../listeners/ListenerConfiguration.java | 5 +- ...udySubmittedVariantsStepConfiguration.java | 12 +- ...ttedVariantsFromFileStepConfiguration.java | 13 +- .../deprecate/MongoTestDatabaseSetup.java | 4 +- .../io/StudySubmittedVariantsReaderTest.java | 49 +-- ...SubmittedVariantsJobConfigurationTest.java | 71 ++--- ...dVariantsFromFileJobConfigurationTest.java | 59 ++-- ...ubmittedVariantsStepConfigurationTest.java | 50 +-- ...VariantsFromFileStepConfigurationTest.java | 49 +-- .../BatchJobRepositoryTestConfiguration.java | 22 ++ .../configuration/BatchTestConfiguration.java | 41 ++- .../configuration/MongoTestConfiguration.java | 70 ++++ .../test/rule/FixSpringMongoDbRule.java | 42 --- ...bmitted-variants-from-file-test.properties | 7 - .../study-submitted-variants-test.properties | 7 - eva-accession-import-dbsnp2/pom.xml | 26 +- .../io/DbsnpJsonClusteredVariantsWriter.java | 23 +- ...DbsnpJsonVariantsStepProgressListener.java | 6 +- .../ContigToGenbankReplacerProcessor.java | 5 +- .../configuration/RunnerConfiguration.java | 13 - ...tDbsnpJsonVariantsWriterConfiguration.java | 4 +- ...portDbsnpJsonVariantsJobConfiguration.java | 7 +- ...ortDbsnpJsonVariantsStepConfiguration.java | 24 +- ...tVariantsJobLauncherCommandLineRunner.java | 22 +- .../DbsnpJsonClusteredVariantsWriterTest.java | 129 ++++---- .../batch/io/DbsnpJsonItemReaderTest.java | 21 +- .../ContigToGenbankReplacerProcessorTest.java | 67 ++-- ...onNodeToClusteredVariantProcessorTest.java | 21 +- ...DbsnpJsonVariantsJobConfigurationTest.java | 23 +- ...bsnpJsonVariantsStepConfigurationTest.java | 26 +- ...iantsJobLauncherCommandLineRunnerTest.java | 73 +++-- .../BatchJobRepositoryTestConfiguration.java | 22 ++ .../dbsnp2/test/BatchTestConfiguration.java | 37 ++- .../dbsnp2/test/MongoTestConfiguration.java | 102 ++++++ .../src/test/resources/application.properties | 3 - .../test-dbsnp-json-import.properties | 0 eva-accession-pipeline/pom.xml | 33 +- .../batch/io/AccessionReportWriter.java | 106 +++---- .../pipeline/batch/io/AccessionWriter.java | 51 +-- .../batch/io/DuplicateSSAccQCWriter.java | 7 +- .../policies/InvalidVariantSkipPolicy.java | 2 +- .../configuration/RunnerConfiguration.java | 13 - .../io/AccessionWriterConfiguration.java | 9 +- .../batch/io/VcfReaderConfiguration.java | 5 +- .../DuplicateSSAccQCJobConfiguration.java | 7 +- .../QCSubsnpAccessionsJobConfiguration.java | 7 +- .../SSAccessionRecoveryJobConfiguration.java | 7 +- .../SubsnpAccessionsJobConfiguration.java | 7 +- .../VariantProcessorConfiguration.java | 3 +- ...AccessioningShutdownStepConfiguration.java | 10 +- .../steps/BuildReportStepConfiguration.java | 16 +- .../DuplicateSSAccQCStepConfiguration.java | 10 +- .../QCSubsnpAccessionsStepConfiguration.java | 10 +- .../SSAccessionRecoveryStepConfiguration.java | 10 +- .../SubsnpAccessionsStepConfiguration.java | 10 +- .../metric/AccessioningMetricCompute.java | 16 +- ...AccessionJobLauncherCommandLineRunner.java | 20 +- .../batch/io/AccessionReportWriterTest.java | 102 +++--- .../batch/io/AccessionWriterTest.java | 111 ++++--- .../InvalidVariantSkipPolicyTest.java | 10 +- .../ContigToGenbankReplacerProcessorTest.java | 10 +- ...xcludeStructuralVariantsProcessorTest.java | 12 +- .../processors/VariantConverterTest.java | 67 ++-- .../buildReport/BuildReportTaskletTest.java | 29 +- .../AccessionWrapperComparatorTest.java | 21 +- .../reportCheck/ReportCheckTaskletTest.java | 7 +- ...fSubsnpAccessionsJobConfigurationTest.java | 26 +- ...eSubsnpAccessionsJobConfigurationTest.java | 28 +- ...reateSubsnpAccessionsRecoverStateTest.java | 45 ++- ...nsRecoveringStateJobConfigurationTest.java | 48 ++- .../DuplicateSSAccQCJobConfigurationTest.java | 55 ++-- ...CSubsnpAccessionsJobConfigurationTest.java | 18 +- ...SubsnpAccessionsStepConfigurationTest.java | 26 +- ...SubsnpAccessionsStepConfigurationTest.java | 22 +- ...ssionJobLauncherCommandLineRunnerTest.java | 50 +-- .../runner/JobFailureBlocksReleasedTest.java | 34 +- .../pipeline/runner/RestartFailedJobTest.java | 47 ++- .../runner/SSAccessionRecoveryTest.java | 22 +- .../BatchJobRepositoryTestConfiguration.java | 22 ++ .../pipeline/test/BatchTestConfiguration.java | 50 ++- .../pipeline/test/FixSpringMongoDbRule.java | 41 --- .../pipeline/test/MongoTestConfiguration.java | 102 ++++++ .../SSAccessionRecoveryTestConfiguration.java | 20 +- ...ccession-pipeline-interval-test.properties | 3 - ...ion-pipeline-recover-state-test.properties | 11 +- ...accession-pipeline-recover-test.properties | 11 +- .../accession-pipeline-test.properties | 4 +- .../duplicate-ss-acc-qc-test.properties | 3 - .../qc-accession-pipeline-test.properties | 7 +- .../ss-accession-recovery.properties | 11 +- eva-accession-release/pom.xml | 29 +- .../assembly/AssemblyNameRetriever.java | 17 +- .../io/VariantMongoAggregationReader.java | 17 +- .../active/AccessionedVariantMongoReader.java | 2 +- .../batch/io/active/VariantContextWriter.java | 39 ++- .../batch/io/contig/ContigMongoReader.java | 32 +- .../release/batch/io/contig/ContigWriter.java | 5 +- .../DeprecatedVariantAccessionWriter.java | 7 +- .../DeprecatedVariantMongoReader.java | 20 +- .../io/merged/MergedVariantContextWriter.java | 3 +- .../io/merged/MergedVariantMongoReader.java | 56 ++-- ...snpMergedDeprecatedVariantMongoReader.java | 10 +- ...EvaMergedDeprecatedVariantMongoReader.java | 10 +- ...ergedDeprecatedVariantAccessionWriter.java | 4 +- .../MergedDeprecatedVariantMongoReader.java | 36 +-- .../MultimapVariantContextWriter.java | 4 +- .../multimap/MultimapVariantMongoReader.java | 10 +- .../listeners/ExcludeVariantsListener.java | 2 +- .../ExcludeInvalidVariantsProcessor.java | 7 +- .../processors/NamedVariantProcessor.java | 32 +- .../VariantToVariantContextProcessor.java | 11 +- .../configuration/RunnerConfiguration.java | 21 -- ...ccessionReleaseFromDBJobConfiguration.java | 7 +- .../DumpRSAccessionsJobConfiguration.java | 11 +- ...ccessionReleaseFromDBJobConfiguration.java | 7 +- .../ReleaseProcessorConfiguration.java | 5 +- ...cessionReleaseFromDBStepConfiguration.java | 10 +- .../DumpRSAccessionsStepConfiguration.java | 18 +- ...cessionReleaseFromDBStepConfiguration.java | 10 +- .../assembly/AssemblyNameRetrieverTest.java | 41 ++- .../AccessionedVariantMongoReaderTest.java | 225 ++++++------- .../io/active/VariantContextWriterTest.java | 143 +++++---- .../io/contig/ContigMongoReaderTest.java | 58 ++-- .../batch/io/contig/ContigWriterTest.java | 62 ++-- .../DeprecatedVariantAccessionWriterTest.java | 21 +- .../DeprecatedVariantMongoReaderTest.java | 74 ++--- .../merged/MergedVariantMongoReaderTest.java | 250 +++++++-------- ...dDeprecatedVariantAccessionWriterTest.java | 24 +- ...ergedDeprecatedVariantMongoReaderTest.java | 60 ++-- ...ontextNucleotideAdditionProcessorTest.java | 44 +-- .../ExcludeInvalidVariantsProcessorTest.java | 18 +- .../processors/NamedVariantProcessorTest.java | 17 +- .../VariantToVariantContextProcessorTest.java | 65 ++-- .../batch/steps/DumpRSAccessionsTest.java | 47 +-- .../steps/RSAccessionsReleaseFromDBTest.java | 47 +-- .../BatchJobRepositoryTestConfiguration.java | 22 ++ .../configuration/MongoTestConfiguration.java | 70 ++++ .../ReleaseFromDBTestConfiguration.java | 66 ++-- .../test/rule/FixSpringMongoDbRule.java | 41 --- .../src/test/resources/application.properties | 3 - .../dump-rs-accession-test.properties | 3 - ...cremental-release-pipeline-test.properties | 3 - .../release-pipeline-test.properties | 3 - eva-accession-ws/pom.xml | 38 ++- .../accession/ws/EvaAccessionApplication.java | 2 +- .../ApplicationConfiguration.java | 36 +-- .../configuration/SwaggerConfiguration.java | 52 +-- .../rest/ClusteredVariantsRestController.java | 85 +++-- .../ws/rest/EvaControllerAdvice.java | 2 +- .../rest/SubmittedVariantsRestController.java | 74 +++-- .../ClusteredVariantsBeaconService.java | 13 +- .../ClusteredVariantHistoryEndPointTest.java | 84 ++--- .../ClusteredVariantsRestControllerTest.java | 299 +++++++++--------- .../SubmittedVariantsRestControllerTest.java | 166 +++++----- .../ws/test/MongoTestConfiguration.java | 105 ++++++ .../accession/ws/test/TestConfiguration.java | 5 +- .../ws/test/rule/FixSpringMongoDbRule.java | 41 --- .../resources/accession-ws-test.properties | 3 - eva-remapping-get-source/pom.xml | 25 +- .../io/DbsnpSubmittedVariantMongoReader.java | 5 +- .../io/EvaSubmittedVariantMongoReader.java | 3 +- .../source/batch/io/VariantContextWriter.java | 14 +- ...ittedVariantToVariantContextProcessor.java | 3 +- .../configuration/RunnerConfiguration.java | 13 - .../io/VariantContextWriterConfiguration.java | 3 +- ...portSubmittedVariantsJobConfiguration.java | 18 +- .../listeners/ListenersConfiguration.java | 5 +- ...ortSubmittedVariantsStepConfiguration.java | 21 +- .../source/parameters/InputParameters.java | 5 +- ...RemappingJobLauncherCommandLineRunner.java | 22 +- .../DbsnpSubmittedVariantMongoReaderTest.java | 47 ++- .../EvaSubmittedVariantMongoReaderTest.java | 48 ++- .../batch/io/VariantContextWriterTest.java | 52 ++- ...ontextNucleotideAdditionProcessorTest.java | 44 +-- ...eAllelesMismatchVariantsProcessorTest.java | 12 +- .../ExcludeMultimapVariantsProcessorTest.java | 16 +- .../VariantToVariantContextProcessorTest.java | 21 +- ...SubmittedVariantsJobConfigurationTest.java | 88 +++--- ...ppingJobLauncherCommandLineRunnerTest.java | 69 ++-- ...cherCommandLineRunnerWithProjectsTest.java | 66 ++-- .../BatchJobRepositoryTestConfiguration.java | 22 ++ .../configuration/BatchTestConfiguration.java | 30 +- .../configuration/MongoTestConfiguration.java | 70 ++++ .../src/test/resources/application.properties | 1 - .../test/resources/with-projects.properties | 1 - eva-remapping-ingest/pom.xml | 31 +- .../io/RemappedSubmittedVariantsWriter.java | 90 +++--- .../configuration/RunnerConfiguration.java | 13 - .../batch/io/VcfReaderConfiguration.java | 3 +- ...mappedVariantsFromVcfJobConfiguration.java | 15 +- ...ngestRemappedFromVcfStepConfiguration.java | 21 +- ...oreRemappingMetadataStepConfiguration.java | 15 +- .../IngestRemappedCommandLineRunner.java | 20 +- .../RemappedSubmittedVariantsWriterTest.java | 77 ++--- ...edVariantsFromVcfJobConfigurationTest.java | 59 ++-- ...tRemappedFromVcfStepConfigurationTest.java | 61 ++-- ...ppedFromVcfStepDbsnpConfigurationTest.java | 62 ++-- .../BatchJobRepositoryTestConfiguration.java | 22 ++ .../configuration/BatchTestConfiguration.java | 30 +- .../configuration/MongoTestConfiguration.java | 70 ++++ .../test/rule/FixSpringMongoDbRule.java | 38 --- .../ingest-remapped-variants-dbsnp.properties | 3 - .../ingest-remapped-variants.properties | 3 - pom.xml | 168 +++++++--- 328 files changed, 6328 insertions(+), 5899 deletions(-) create mode 100644 eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/test/configuration/BatchJobRepositoryTestConfiguration.java delete mode 100644 eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/test/rule/FixSpringMongoDbRule.java create mode 100644 eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/test/configuration/JPATestConfiguration.java delete mode 100644 eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/test/rule/FixSpringMongoDbRule.java create mode 100644 eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/utils/MongoTestContainerHelper.java create mode 100644 eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/utils/MongoTestDataLoader.java create mode 100644 eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/utils/PipelineTemporaryFolderUtil.java create mode 100644 eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/test/configuration/BatchJobRepositoryTestConfiguration.java delete mode 100644 eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/test/rule/FixSpringMongoDbRule.java create mode 100644 eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/test/BatchJobRepositoryTestConfiguration.java create mode 100644 eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/test/MongoTestConfiguration.java delete mode 100644 eva-accession-import-dbsnp2/src/test/resources/test-dbsnp-json-import.properties create mode 100644 eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/test/BatchJobRepositoryTestConfiguration.java delete mode 100644 eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/test/FixSpringMongoDbRule.java create mode 100644 eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/test/MongoTestConfiguration.java create mode 100644 eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/test/configuration/BatchJobRepositoryTestConfiguration.java delete mode 100644 eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/test/rule/FixSpringMongoDbRule.java create mode 100644 eva-accession-ws/src/test/java/uk/ac/ebi/eva/accession/ws/test/MongoTestConfiguration.java delete mode 100644 eva-accession-ws/src/test/java/uk/ac/ebi/eva/accession/ws/test/rule/FixSpringMongoDbRule.java create mode 100644 eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/test/configuration/BatchJobRepositoryTestConfiguration.java create mode 100644 eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/test/configuration/BatchJobRepositoryTestConfiguration.java delete mode 100644 eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/test/rule/FixSpringMongoDbRule.java diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 6bd664e22..ed97e97fd 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -4,42 +4,26 @@ on: push: branches: [ master, java21 ] pull_request: - # Temporarily run tests on the feature branch until merged to master - branches: [ master, java21, feature/ws-contig-alias-integration ] + branches: [ master, java21 ] jobs: build: runs-on: ubuntu-latest + strategy: matrix: - # Including "ubuntu2204" in the version is a temporary fix to the direct download URL while we transition - # between 4 and 6 - mongodb-version: [4.0.18, ubuntu2204-6.0.6] - java-version: [1.8] - python-version: [3.8] + java-version: [21] steps: - - uses: actions/checkout@v2 - - name: Setup Java JDK - uses: actions/setup-java@v1.4.3 - # The Java version to make available on the path. Takes a whole or semver Java version, or 1.x syntax (e.g. 1.8 => Java 8.x). Early access versions can be specified in the form of e.g. 14-ea, 14.0.0-ea, or 14.0.0-ea.28 - with: - java-version: ${{ matrix.java-version }} - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Java 21 + uses: actions/setup-java@v4 with: - python-version: ${{ matrix.python-version }} - - name: Install Python dependencies - run: | - python -m pip install --upgrade pip - pip install numpy pandas psycopg2-binary pgpasslib - - name: Install and Start MongoDB - run: | - wget https://fastdl.mongodb.org/linux/mongodb-linux-x86_64-${{ matrix.mongodb-version }}.tgz - tar xfz mongodb-linux-x86_64-${{ matrix.mongodb-version }}.tgz - export PATH=`pwd`/mongodb-linux-x86_64-${{ matrix.mongodb-version }}/bin:$PATH - mkdir -p data/db - mongod --dbpath=data/db & - mongod --version + distribution: temurin + java-version: ${{ matrix.java-version }} + cache: maven + - name: Run Maven tests - run: mvn test + run: mvn -B clean test \ No newline at end of file diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 69b6eda0f..02f0123f0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -28,7 +28,7 @@ test: - mongo:4.0.18 script: # Gitlab exposes services under their own hostnames. So test host should be "mongo" instead of "localhost". - - mvn clean test --projects 'eva-accession-core,eva-accession-deprecate,eva-accession-ws,eva-remapping-get-source,eva-accession-clustering,eva-accession-pipeline,eva-accession-release,eva-remapping-ingest' -Deva.mongo.host.test=mongo + - mvn clean test --projects 'eva-accession-core,eva-accession-deprecate,eva-accession-ws,eva-remapping-get-source,eva-accession-clustering,eva-accession-pipeline,eva-accession-release,eva-remapping-ingest' environment: name: test-env only: diff --git a/eva-accession-clustering/pom.xml b/eva-accession-clustering/pom.xml index 026d09201..53ac31e80 100644 --- a/eva-accession-clustering/pom.xml +++ b/eva-accession-clustering/pom.xml @@ -2,12 +2,13 @@ + 4.0.0 + eva-accession uk.ac.ebi.eva - 0.6.58-SNAPSHOT + ${revision} - 4.0.0 eva-accession-clustering jar @@ -31,9 +32,8 @@ - com.google.guava - guava - 33.4.0-jre + com.google.code.gson + gson @@ -41,6 +41,14 @@ xstream + + uk.ac.ebi.eva + eva-accession-core + ${revision} + test-jar + test + + org.springframework.batch spring-batch-test @@ -48,8 +56,24 @@ - com.lordofthejars - nosqlunit-mongodb + org.springframework.boot + spring-boot-starter-test + test + + + + org.springframework.boot + spring-boot-testcontainers + test + + + org.testcontainers + mongodb + test + + + org.testcontainers + junit-jupiter test @@ -76,9 +100,19 @@ - src/test/resources/ + src/test/resources + + **/*.gz + true + + src/test/resources + + **/*.gz + + false + @@ -90,8 +124,4 @@ - - localhost - - diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/io/BackPropagatedRSWriter.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/io/BackPropagatedRSWriter.java index 32ddc983c..0f9b8a1e1 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/io/BackPropagatedRSWriter.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/io/BackPropagatedRSWriter.java @@ -16,13 +16,13 @@ package uk.ac.ebi.eva.accession.clustering.batch.io; import com.mongodb.MongoBulkWriteException; - +import jakarta.annotation.Nonnull; +import org.springframework.batch.item.Chunk; import org.springframework.batch.item.ItemWriter; import org.springframework.data.mongodb.core.BulkOperations; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.data.mongodb.core.query.Query; import org.springframework.data.mongodb.core.query.Update; - import uk.ac.ebi.ampt2d.commons.accession.core.exceptions.AccessionCouldNotBeGeneratedException; import uk.ac.ebi.ampt2d.commons.accession.core.models.EventType; import uk.ac.ebi.ampt2d.commons.accession.persistence.mongodb.document.AccessionedDocument; @@ -36,7 +36,6 @@ import uk.ac.ebi.eva.accession.core.service.nonhuman.SubmittedVariantAccessioningService; import uk.ac.ebi.eva.metrics.metric.MetricCompute; -import javax.annotation.Nonnull; import java.util.Collections; import java.util.Comparator; import java.util.List; @@ -74,11 +73,10 @@ public BackPropagatedRSWriter(String remappedAssembly, ClusteringWriter clusteri } @Override - public void write( - @Nonnull List submittedVariantEntitiesInOriginalAssembly) + public void write(@Nonnull Chunk submittedVariantEntitiesInOriginalAssembly) throws MongoBulkWriteException, AccessionCouldNotBeGeneratedException { List ssToLookupInRemappedAssembly = - submittedVariantEntitiesInOriginalAssembly + submittedVariantEntitiesInOriginalAssembly.getItems() .stream() // Some dbSNP imported variants might have been explicitly de-clustered // because the REF/ALT allele data provided by dbSNP was internally inconsistent @@ -89,15 +87,15 @@ public void write( .collect(Collectors.toList()); List ssIDsToLookupInRemappedAssembly = ssToLookupInRemappedAssembly.stream().map(AccessionedDocument::getAccession) - .collect(Collectors.toList()); + .collect(Collectors.toList()); Map> ssInRemappedAssemblyGroupedByID = submittedVariantAccessioningService .getAllActiveByAssemblyAndAccessionIn(remappedAssembly, ssIDsToLookupInRemappedAssembly) .stream() .map(entity -> new SubmittedVariantEntity(entity.getAccession(), entity.getHash(), - entity.getData(), - entity.getVersion())) + entity.getData(), + entity.getVersion())) .filter(entity -> Objects.nonNull(entity.getClusteredVariantAccession())) .collect(Collectors.groupingBy(SubmittedVariantEntity::getAccession)); backpropagateRSToSS(ssToLookupInRemappedAssembly, ssInRemappedAssemblyGroupedByID); @@ -109,21 +107,21 @@ private void backpropagateRSToSS( // Updates to Submitted Variant Entity (SVE) collection with the RS created above BulkOperations evaSVEUpdates = mongoTemplate.bulkOps(BulkOperations.BulkMode.UNORDERED, - SubmittedVariantEntity.class); + SubmittedVariantEntity.class); BulkOperations dbsnpSVEUpdates = mongoTemplate.bulkOps(BulkOperations.BulkMode.UNORDERED, - DbsnpSubmittedVariantEntity.class); + DbsnpSubmittedVariantEntity.class); // Inserts to Submitted Variant Operation Entity (SVOE) collection recording the RS updates made above BulkOperations evaSVOEInserts = mongoTemplate.bulkOps(BulkOperations.BulkMode.UNORDERED, - SubmittedVariantOperationEntity.class); + SubmittedVariantOperationEntity.class); BulkOperations dbsnpSVOEInserts = mongoTemplate.bulkOps(BulkOperations.BulkMode.UNORDERED, - DbsnpSubmittedVariantOperationEntity.class); + DbsnpSubmittedVariantOperationEntity.class); int numEvaRSAssignments = 0; int numDbsnpRSAssignments = 0; - for (SubmittedVariantEntity submittedVariantEntity: submittedVariantEntitiesInOriginalAssemblyWithNoRS) { + for (SubmittedVariantEntity submittedVariantEntity : submittedVariantEntitiesInOriginalAssemblyWithNoRS) { Long ssIDToBeClustered = submittedVariantEntity.getAccession(); Long rsInOriginalAssembly = submittedVariantEntity.getClusteredVariantAccession(); Long rsToBackPropagate = null; diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/io/ClusteringWriter.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/io/ClusteringWriter.java index 6bc7b8617..adc0ba730 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/io/ClusteringWriter.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/io/ClusteringWriter.java @@ -17,13 +17,14 @@ import com.mongodb.MongoBulkWriteException; import htsjdk.samtools.util.StringUtil; +import jakarta.annotation.Nonnull; import org.springframework.batch.core.JobExecution; +import org.springframework.batch.item.Chunk; import org.springframework.batch.item.ItemWriter; import org.springframework.data.mongodb.core.BulkOperations; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.data.mongodb.core.query.Query; import org.springframework.data.mongodb.core.query.Update; -import org.springframework.data.util.CloseableIterator; import org.springframework.util.Assert; import uk.ac.ebi.ampt2d.commons.accession.core.exceptions.AccessionCouldNotBeGeneratedException; import uk.ac.ebi.ampt2d.commons.accession.core.exceptions.AccessionDoesNotExistException; @@ -32,7 +33,6 @@ import uk.ac.ebi.ampt2d.commons.accession.core.models.GetOrCreateAccessionWrapper; import uk.ac.ebi.ampt2d.commons.accession.hashing.SHA1HashingFunction; import uk.ac.ebi.ampt2d.commons.accession.persistence.mongodb.document.EventDocument; - import uk.ac.ebi.eva.accession.clustering.metric.ClusteringMetric; import uk.ac.ebi.eva.accession.core.EVAObjectModelUtils; import uk.ac.ebi.eva.accession.core.model.ClusteredVariant; @@ -53,7 +53,6 @@ import uk.ac.ebi.eva.commons.core.models.VariantClassifier; import uk.ac.ebi.eva.metrics.metric.MetricCompute; -import javax.annotation.Nonnull; import java.io.File; import java.io.FileWriter; import java.io.IOException; @@ -68,16 +67,18 @@ import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; +import java.util.stream.Stream; import static org.springframework.data.mongodb.core.query.Criteria.where; import static org.springframework.data.mongodb.core.query.Query.query; -import static uk.ac.ebi.eva.accession.clustering.configuration.batch.io.RSMergeAndSplitCandidatesReaderConfiguration.*; +import static uk.ac.ebi.eva.accession.clustering.configuration.batch.io.RSMergeAndSplitCandidatesReaderConfiguration.MERGE_CANDIDATE_ID_PREFIX; +import static uk.ac.ebi.eva.accession.clustering.configuration.batch.io.RSMergeAndSplitCandidatesReaderConfiguration.SPLIT_CANDIDATE_ID_PREFIX; /** * This writer has two parts: * 1. Use the accessioning service to generate new RS IDs or get existing ones * 2. Update the submitted variants to include the "rs" field with the generated/retrieved accessions - * + *

* Some edge cases take into account if a clustered variant is multimap. The definition of multimap variants that this * class uses is "clustered variants whose mapWeight is 2 or greater". Another definition is "clustered variants * whose accession maps several times in the same assembly". Although both definitions should yield the same @@ -141,16 +142,15 @@ public ClusteringWriter(MongoTemplate mongoTemplate, private void getSVOEWithMergeAndRSSplitCandidates() { List MERGE_AND_SPLIT_EVENTS = Arrays.asList(EventType.RS_MERGE_CANDIDATES.name(), - EventType.RS_SPLIT_CANDIDATES.name()); + EventType.RS_SPLIT_CANDIDATES.name()); Query queryOperations = query(where("eventType").in(MERGE_AND_SPLIT_EVENTS) - .and("inactiveObjects").elemMatch(where("seq").is(assembly))); + .and("inactiveObjects").elemMatch(where("seq").is(assembly))); mergeCandidateSVOE = new HashMap<>(); rsSplitCandidateSVOE = new HashMap<>(); - try ( - CloseableIterator svoes = mongoTemplate.stream( - queryOperations, SubmittedVariantOperationEntity.class)) { - svoes.forEachRemaining(svoe -> { + try (Stream svoes = mongoTemplate.stream( + queryOperations, SubmittedVariantOperationEntity.class)) { + svoes.forEach(svoe -> { if (svoe.getEventType().equals(EventType.RS_MERGE_CANDIDATES)) { mergeCandidateSVOE.put(getClusteredVariantHash(svoe.getInactiveObjects().get(0).getModel()), svoe); } else if (svoe.getEventType().equals(EventType.RS_SPLIT_CANDIDATES)) { @@ -161,7 +161,7 @@ private void getSVOEWithMergeAndRSSplitCandidates() { } @Override - public void write(@Nonnull List submittedVariantEntities) + public void write(@Nonnull Chunk submittedVariantEntities) throws IOException, MongoBulkWriteException, AccessionCouldNotBeGeneratedException, AccessionDoesNotExistException { try { @@ -175,27 +175,26 @@ public void write(@Nonnull List submittedVaria if (!this.processClusteredRemappedVariants) { clusterSubmittedVariants(submittedVariantEntities); } - } - finally { + } finally { this.rsReportFileWriter.close(); } } - private void getOrCreateClusteredVariantAccessions(List submittedVariantEntities) + private void getOrCreateClusteredVariantAccessions(Chunk submittedVariantEntities) throws AccessionCouldNotBeGeneratedException, IOException { if (processClusteredRemappedVariants) { processClusteredRemappedVariants(submittedVariantEntities); } else { - List clusteredVariants = submittedVariantEntities.stream() - .map(this::toClusteredVariant) - .collect(Collectors.toList()); + List clusteredVariants = submittedVariantEntities.getItems().stream() + .map(this::toClusteredVariant) + .collect(Collectors.toList()); if (!clusteredVariants.isEmpty()) { List> accessionWrappers = clusteredService.getOrCreate(clusteredVariants, jobExecution.getJobId().toString()); for (GetOrCreateAccessionWrapper result : accessionWrappers) { if (result.isNewAccession()) { ClusteringWriter.writeRSReportEntry(this.rsReportFileWriter, result.getAccession(), - result.getHash()); + result.getHash()); } } List> accessionNoMultimap = @@ -216,28 +215,28 @@ private void getOrCreateClusteredVariantAccessions(List * --------------------Before Clustering-------------------- - * - * SubmittedVariantEntity + *

+ * SubmittedVariantEntity * SS RS ASM STUDY CONTIG POS REF ALT * 500 306 ASM1 PRJEB1 Chr1 1000 A T (original) * 501 306 ASM1 PRJEB2 Chr1 1000 A T (original) * 500 306 ASM2 PRJEB1 Chr1 1000 A T (remapped) * 501 306 ASM2 PRJEB2 Chr1 1500 A T (remapped) - * - * ClusteredVariantEntity + *

+ * ClusteredVariantEntity * RS HASH ASM POS CONTIG TYPE * 306 ASM1_Chr1_1000_SNV ASM1 1000 Chr1 SNV - * + *

* SS id 500 and 501 has same RS because of remapping, but they are now at different positions and can't have same RS id. * These needs to be identified and stored in submittedVariantOperationEntity table for rectification. - * + *

* --------------------After Detection-------------------- - * - * SubmittedVariantOperationEntity + *

+ * SubmittedVariantOperationEntity * ACCESSION EVENT_TYPE REASON INACTIVe_OBJECTS * 306 RS_SPLIT_CANDIDATE Hash mismatch with 306 {ss-500 and ss-501} */ - private void processClusteredRemappedVariants(List submittedVariants) + private void processClusteredRemappedVariants(Chunk submittedVariants) throws IOException { List clusteredRemappedSubmittedVariants = getClusteredAndRemappedVariants(submittedVariants); if (clusteredRemappedSubmittedVariants.isEmpty()) { @@ -249,8 +248,8 @@ private void processClusteredRemappedVariants(List updateRsSplitCandidateSVOE = new HashMap<>(); Set rsIDs = clusteredRemappedSubmittedVariants.stream() - .map(SubmittedVariantEntity::getClusteredVariantAccession) - .collect(Collectors.toSet()); + .map(SubmittedVariantEntity::getClusteredVariantAccession) + .collect(Collectors.toSet()); // Initialize a map of "RS -> list of associated hashes" with hashes for RS IDs already present in the database // We will keep updating this map when we encounter new RS IDs present in the remapped variants SS record // but with no corresponding RS record in the database @@ -262,12 +261,12 @@ private void processClusteredRemappedVariants(List()); checkIfCandidateForRSSplit(remappedSubmittedVariantEntity, clusteredVariantEntity, - allExistingHashesGroupByRS, updateRsSplitCandidateSVOE); + allExistingHashesGroupByRS, updateRsSplitCandidateSVOE); if (!isExistingRSOrMergeCandidate) { if (clusteredVariantEntity.getAccession() >= accessioningMonotonicInitRs) { @@ -276,17 +275,17 @@ private void processClusteredRemappedVariants(List getClusteredAndRemappedVariants(List submittedVariants) { - return submittedVariants.stream() + private List getClusteredAndRemappedVariants(Chunk submittedVariants) { + return submittedVariants.getItems().stream() .filter(sve -> Objects.nonNull(sve.getClusteredVariantAccession())) .filter(sve -> !StringUtil.isBlank(sve.getRemappedFrom())) .filter(sve -> Objects.isNull(sve.getMapWeight())) @@ -298,7 +297,7 @@ private Map getSubmittedVariantsAllExistingHashesInDB(List> getAllHashesForAssemblyAndRSInDB(List accessionList) { @@ -353,9 +352,9 @@ private boolean checkIfExistingRSOrCandidateForMerge(SubmittedVariantEntity subm } else { List inactiveObjects = getAllSubmittedVariantsWithClusteringAccession(accessionInDB).stream() - .filter(sve-> toClusteredVariantEntity(sve).getHashedMessage().equals(variantHash)) - .map(SubmittedVariantInactiveEntity::new) - .collect(Collectors.toList()); + .filter(sve -> toClusteredVariantEntity(sve).getHashedMessage().equals(variantHash)) + .map(SubmittedVariantInactiveEntity::new) + .collect(Collectors.toList()); inactiveObjects.add(new SubmittedVariantInactiveEntity(submittedVariantEntity)); submittedVariantOperationEntity = new SubmittedVariantOperationEntity(); submittedVariantOperationEntity.fill(EventType.RS_MERGE_CANDIDATES, accessionInDB, @@ -395,8 +394,8 @@ private void checkIfCandidateForRSSplit(SubmittedVariantEntity submittedVariantE submittedVariantOperationEntity = new SubmittedVariantOperationEntity(); List inactiveEntities = getAllSubmittedVariantsWithClusteringAccession(variantAccession).stream() - .map(SubmittedVariantInactiveEntity::new) - .collect(Collectors.toList()); + .map(SubmittedVariantInactiveEntity::new) + .collect(Collectors.toList()); submittedVariantOperationEntity.fill(EventType.RS_SPLIT_CANDIDATES, variantAccession, "Hash mismatch with " + variantAccession, inactiveEntities); rsSplitCandidateSVOE.put(variantAccession, submittedVariantOperationEntity); @@ -433,7 +432,7 @@ private void insertAllEntriesInDB(List clusteredVariantE for (Map.Entry entry : mergeSVOE.entrySet()) { SubmittedVariantOperationEntity svoe = entry.getValue(); - if(Objects.isNull(svoe.getId())){ + if (Objects.isNull(svoe.getId())) { svoe.setId(getMergeCandidateId(svoe)); mergeSVOEInsertEntries.add(svoe); continue; @@ -451,7 +450,7 @@ private void insertAllEntriesInDB(List clusteredVariantE for (Map.Entry entry : rsSplitSVOE.entrySet()) { Long accession = entry.getKey(); SubmittedVariantOperationEntity svoe = entry.getValue(); - if(Objects.isNull(svoe.getId())){ + if (Objects.isNull(svoe.getId())) { svoe.setId(getSplitCandidateId(svoe)); rsSplitSVOEInsertEntries.add(svoe); continue; @@ -478,13 +477,13 @@ public ClusteredVariantEntity toClusteredVariantEntity(SubmittedVariantEntity su private ClusteredVariant toClusteredVariant(ISubmittedVariant submittedVariant) { return new ClusteredVariant(submittedVariant.getReferenceSequenceAccession(), - submittedVariant.getTaxonomyAccession(), - submittedVariant.getContig(), - submittedVariant.getStart(), - VariantClassifier.getVariantClassification(submittedVariant.getReferenceAllele(), - submittedVariant.getAlternateAllele()), - submittedVariant.isValidated(), - submittedVariant.getCreatedDate()); + submittedVariant.getTaxonomyAccession(), + submittedVariant.getContig(), + submittedVariant.getStart(), + VariantClassifier.getVariantClassification(submittedVariant.getReferenceAllele(), + submittedVariant.getAlternateAllele()), + submittedVariant.isValidated(), + submittedVariant.getCreatedDate()); } public Class> @@ -505,7 +504,7 @@ private boolean isEvaClusteredAccession(Long accession) { * In EVA-2003 we decided not to merge any RS ID that maps to several places in the same assembly * (mapping weight > 1) as this might be a signal of a low quality variant, and merging other "real" variants * in the same RS would just make things more complicated. - * + *

* Note that for submitted variants the test is not this simple, as 1:1000:A:T and 1:1000:A:G can be present in the * same assembly and still not classify as multimap. */ @@ -519,7 +518,7 @@ protected boolean isMultimap(IClusteredVariant clusteredVariant) { /** * From EVA-2071, do not cluster submitted variants into a multimap clustered variant. - * + *

* This function removes candidate clustered variant accessions if they are multimap. This means that some submitted * variants will be kept unclustered. This potentially will be revisited in the future, but for now (release 2) we * are leaving this out of scope. @@ -532,11 +531,11 @@ private List> exclu /** * This function assigns a clustered variant accession (rs) to the submitted variants that didn't have any. */ - private void clusterSubmittedVariants(List submittedVariantEntities) { + private void clusterSubmittedVariants(Chunk submittedVariantEntities) { BulkOperations bulkOperations = mongoTemplate.bulkOps(BulkOperations.BulkMode.UNORDERED, - SubmittedVariantEntity.class); + SubmittedVariantEntity.class); BulkOperations dbsnpBulkOperations = mongoTemplate.bulkOps(BulkOperations.BulkMode.UNORDERED, - DbsnpSubmittedVariantEntity.class); + DbsnpSubmittedVariantEntity.class); BulkOperations bulkHistoryOperations = mongoTemplate.bulkOps(BulkOperations.BulkMode.UNORDERED, SubmittedVariantOperationEntity.class); BulkOperations dbsnpBulkHistoryOperations = mongoTemplate.bulkOps(BulkOperations.BulkMode.UNORDERED, @@ -609,21 +608,21 @@ protected String getClusteredVariantHash(ISubmittedVariant submittedVariant) { return clusteredHashingFunction.apply(clusteredVariant); } - protected static void writeRSReportEntry (FileWriter rsReportWriter, Long rsAccession, String rsHash) + protected static void writeRSReportEntry(FileWriter rsReportWriter, Long rsAccession, String rsHash) throws IOException { rsReportWriter.write(String.format("%s\t%s\n", rsAccession, rsHash)); } public static String getMergeCandidateId(SubmittedVariantOperationEntity svoe) { return String.format("%s_%s_%s", MERGE_CANDIDATE_ID_PREFIX, - svoe.getInactiveObjects().get(0).getReferenceSequenceAccession(), - EVAObjectModelUtils.getClusteredVariantHash(svoe.getInactiveObjects().get(0))); + svoe.getInactiveObjects().get(0).getReferenceSequenceAccession(), + EVAObjectModelUtils.getClusteredVariantHash(svoe.getInactiveObjects().get(0))); } public static String getSplitCandidateId(SubmittedVariantOperationEntity svoe) { return String.format("%s_%s_%d", SPLIT_CANDIDATE_ID_PREFIX, - svoe.getInactiveObjects().get(0).getReferenceSequenceAccession(), - svoe.getInactiveObjects().get(0).getClusteredVariantAccession()); + svoe.getInactiveObjects().get(0).getReferenceSequenceAccession(), + svoe.getInactiveObjects().get(0).getClusteredVariantAccession()); } public void setJobExecution(JobExecution jobExecution) { diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/io/ListOfListItemWriter.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/io/ListOfListItemWriter.java index f4c270667..ee8479bb2 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/io/ListOfListItemWriter.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/io/ListOfListItemWriter.java @@ -15,6 +15,7 @@ */ package uk.ac.ebi.eva.accession.clustering.batch.io; +import org.springframework.batch.item.Chunk; import org.springframework.batch.item.ItemStreamWriter; import org.springframework.batch.item.ItemWriter; @@ -32,14 +33,14 @@ public ListOfListItemWriter(ItemWriter itemWriter) { } @Override - public void write(List> listOfLists) throws Exception { + public void write(Chunk> listOfLists) throws Exception { if (listOfLists.isEmpty()) { return; } - List all = listOfLists.stream().flatMap(Collection::stream).collect(Collectors.toList()); + List all = listOfLists.getItems().stream().flatMap(Collection::stream).collect(Collectors.toList()); - itemWriter.write(all); + itemWriter.write(new Chunk<>(all)); } public void setItemWriter(ItemStreamWriter itemWriter) { diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/io/RSMergeWriter.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/io/RSMergeWriter.java index 7996c8e2b..af6a780b6 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/io/RSMergeWriter.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/io/RSMergeWriter.java @@ -16,9 +16,11 @@ package uk.ac.ebi.eva.accession.clustering.batch.io; import com.mongodb.MongoBulkWriteException; +import jakarta.annotation.Nonnull; import org.apache.commons.lang3.tuple.ImmutablePair; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.batch.item.Chunk; import org.springframework.batch.item.ItemWriter; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.data.mongodb.core.query.Criteria; @@ -45,7 +47,6 @@ import uk.ac.ebi.eva.commons.core.models.contigalias.ContigNamingConvention; import uk.ac.ebi.eva.metrics.metric.MetricCompute; -import javax.annotation.Nonnull; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -126,9 +127,9 @@ public RSMergeWriter(ClusteringWriter clusteringWriter, MongoTemplate mongoTempl } @Override - public void write(@Nonnull List submittedVariantOperationEntities) + public void write(@Nonnull Chunk submittedVariantOperationEntities) throws MongoBulkWriteException, AccessionCouldNotBeGeneratedException, AccessionDoesNotExistException { - allMergeCandidateOperations = new ArrayList<>(submittedVariantOperationEntities); + allMergeCandidateOperations = new ArrayList<>(submittedVariantOperationEntities.getItems()); // Create a map of all merge candidate operations keyed by RS ID to facilitate quick random lookups populateOperationsIndex(allMergeCandidateOperations); for (int i = 0; i < allMergeCandidateOperations.size(); i++) { @@ -136,9 +137,9 @@ public void write(@Nonnull List submi writeRSMerge(allMergeCandidateOperations.get(i)); } List allCandidateIds = allMergeCandidateOperations.stream().map(EventDocument::getId) - .collect(Collectors.toList()); + .collect(Collectors.toList()); this.mongoTemplate.findAllAndRemove(query(where("_id").in(allCandidateIds)), - SubmittedVariantOperationEntity.class); + SubmittedVariantOperationEntity.class); } private void populateOperationsIndex @@ -146,10 +147,10 @@ public void write(@Nonnull List submi rsIDIndexedOperations = new HashMap<>(); for (int i = 0; i < allMergeCandidateOperations.size(); i++) { SubmittedVariantOperationEntity operation = allMergeCandidateOperations.get(i); - for (Long rsID: + for (Long rsID : operation.getInactiveObjects().stream() - .map(SubmittedVariantInactiveEntity::getClusteredVariantAccession) - .collect(Collectors.toSet())) { + .map(SubmittedVariantInactiveEntity::getClusteredVariantAccession) + .collect(Collectors.toSet())) { OperationWithIndex obj = new OperationWithIndex(i, operation); if (rsIDIndexedOperations.containsKey(rsID)) { rsIDIndexedOperations.get(rsID).add(obj); @@ -176,13 +177,13 @@ public void writeRSMerge(SubmittedVariantOperationEntity currentOperation) // because ClusteredVariantEntity "equals" method does NOT involve comparing accessions List mergeCandidates = currentOperation.getInactiveObjects() - .stream() - // Ensure duplicates inside inactiveObjects are tolerated - .filter(distinctByKey(this::getHashedMessageAndAccessionForSVIE)) - .map(entity -> clusteringWriter.toClusteredVariantEntity( - entity.toSubmittedVariantEntity())) - .filter(distinctByKey(AccessionedDocument::getAccession)) - .collect(Collectors.toList()); + .stream() + // Ensure duplicates inside inactiveObjects are tolerated + .filter(distinctByKey(this::getHashedMessageAndAccessionForSVIE)) + .map(entity -> clusteringWriter.toClusteredVariantEntity( + entity.toSubmittedVariantEntity())) + .filter(distinctByKey(AccessionedDocument::getAccession)) + .collect(Collectors.toList()); // From among the participating RS in a merge, // use the current RS prioritization policy to get the target RS into which the rest of the RS will be merged ImmutablePair> mergeDestinationAndMergees = @@ -192,9 +193,9 @@ public void writeRSMerge(SubmittedVariantOperationEntity currentOperation) removeMergeesAndInsertMergeDestination(mergeDestination, mergees); - for (ClusteredVariantEntity mergee: mergees) { + for (ClusteredVariantEntity mergee : mergees) { logger.info("RS merge operation: Merging rs{} to rs{} due to hash collision...", - mergee.getAccession(), mergeDestination.getAccession()); + mergee.getAccession(), mergeDestination.getAccession()); recordMergeOperations(mergeDestination, mergee, currentOperation); } } @@ -234,17 +235,17 @@ private void insertMergeOperation(ClusteredVariantEntity mergeDestination, Clust List> existingOperations = this.mongoTemplate.find(query(where(ACCESSION_ATTRIBUTE).is(mergee.getAccession())) - .addCriteria(where(MERGE_DESTINATION_ATTRIBUTE).is( - mergeDestination.getAccession())) - .addCriteria(where(ASM_ATTRIBUTE_IN_OPERATIONS_COLLECTION) - .is(this.assemblyAccession)), - operationsCollectionToWriteTo); + .addCriteria(where(MERGE_DESTINATION_ATTRIBUTE).is( + mergeDestination.getAccession())) + .addCriteria(where(ASM_ATTRIBUTE_IN_OPERATIONS_COLLECTION) + .is(this.assemblyAccession)), + operationsCollectionToWriteTo); if (existingOperations.isEmpty()) { ClusteredVariantOperationEntity operation = new ClusteredVariantOperationEntity(); operation.fill(EventType.MERGED, mergee.getAccession(), mergeDestination.getAccession(), - "After remapping to " + mergee.getAssemblyAccession() + - ", RS IDs mapped to the same locus.", - Collections.singletonList(new ClusteredVariantInactiveEntity(mergee))); + "After remapping to " + mergee.getAssemblyAccession() + + ", RS IDs mapped to the same locus.", + Collections.singletonList(new ClusteredVariantInactiveEntity(mergee))); this.mongoTemplate.insert(operation, this.mongoTemplate.getCollectionName(operationsCollectionToWriteTo)); metricCompute.addCount(ClusteringMetric.CLUSTERED_VARIANTS_MERGE_OPERATIONS, 1); @@ -261,6 +262,7 @@ private void insertRSRecordForMergeDestination(ClusteredVariantEntity mergeDesti /** * Get destination RS and the set of RS to be merged into the destination RS + * * @param mergeCandidates Set of RS candidates that should be merged * @return Pair with the first element being the destination RS * and the next being the list of RS that should be merged into the former. @@ -274,17 +276,17 @@ private ImmutablePair> getM } final Long targetRSAccession = lastPrioritizedAccession; ClusteredVariantEntity targetRS = mergeCandidates.stream().filter(rs -> rs.getAccession() - .equals(targetRSAccession)) - .findFirst().get(); + .equals(targetRSAccession)) + .findFirst().get(); List mergees = mergeCandidates.stream() - .filter(rs -> !rs.getAccession() - .equals(targetRSAccession)) - .collect(Collectors.toList()); + .filter(rs -> !rs.getAccession() + .equals(targetRSAccession)) + .collect(Collectors.toList()); return new ImmutablePair<>(targetRS, mergees); } protected void recordMergeOperations(ClusteredVariantEntity mergeDestination, ClusteredVariantEntity mergee, - SubmittedVariantOperationEntity currentOperation) { + SubmittedVariantOperationEntity currentOperation) { Long accessionToBeMerged = mergee.getAccession(); Long accessionToKeep = mergeDestination.getAccession(); @@ -296,9 +298,9 @@ protected void recordMergeOperations(ClusteredVariantEntity mergeDestination, Cl ClusteredVariantMergingPolicy.Priority prioritised = new ClusteredVariantMergingPolicy.Priority(accessionToKeep, accessionToBeMerged); updateSubmittedVariants(prioritised, currentOperation, SubmittedVariantEntity.class, - SubmittedVariantOperationEntity.class); + SubmittedVariantOperationEntity.class); updateSubmittedVariants(prioritised, currentOperation, DbsnpSubmittedVariantEntity.class, - DbsnpSubmittedVariantOperationEntity.class); + DbsnpSubmittedVariantOperationEntity.class); // Update other merge candidate operations involving the mergee // by replacing references to mergee RS ID with the target RS ID @@ -325,36 +327,36 @@ private void updateOperationsInDB(ClusteredVariantMergingPolicy.Priority priorit where(RS_KEY_IN_OPERATIONS_COLLECTION).is(prioritised.accessionToBeMerged))); List operationsInDBInvolvingMergee = mongoTemplate.find(queryForMergeCandidatesInvolvingMergee, SubmittedVariantOperationEntity.class); - for (SubmittedVariantOperationEntity operation: operationsInDBInvolvingMergee) { + for (SubmittedVariantOperationEntity operation : operationsInDBInvolvingMergee) { List submittedVariantInactiveEntitiesWithMergeeRSReplaced = operation.getInactiveObjects().stream() - .map(entity -> replaceRSInSubmittedVariantInactiveEntity( - entity, prioritised.accessionToBeMerged, prioritised.accessionToKeep) - ).collect(Collectors.toList()); + .map(entity -> replaceRSInSubmittedVariantInactiveEntity( + entity, prioritised.accessionToBeMerged, prioritised.accessionToKeep) + ).collect(Collectors.toList()); mongoTemplate.updateFirst(query(where(ID_ATTRIBUTE).is(operation.getId())), - update(INACTIVE_OBJECT_ATTRIBUTE, - submittedVariantInactiveEntitiesWithMergeeRSReplaced), - SubmittedVariantOperationEntity.class); + update(INACTIVE_OBJECT_ATTRIBUTE, + submittedVariantInactiveEntitiesWithMergeeRSReplaced), + SubmittedVariantOperationEntity.class); } } private void updateOperationsInMemory(ClusteredVariantMergingPolicy.Priority prioritised, - SubmittedVariantOperationEntity currentMergeOperation) { + SubmittedVariantOperationEntity currentMergeOperation) { if (rsIDIndexedOperations.containsKey(prioritised.accessionToBeMerged)) { List operationsInMemoryInvolvingMergee = rsIDIndexedOperations.get(prioritised.accessionToBeMerged) - .stream() - // Filter for upcoming operations only - .filter(e -> (e.operationIndex > this.currentlyProcessingOperationIndex) - && !(e.operation.equals(currentMergeOperation))) - .collect(Collectors.toList()); - for (OperationWithIndex operationWithIndex: operationsInMemoryInvolvingMergee) { + .stream() + // Filter for upcoming operations only + .filter(e -> (e.operationIndex > this.currentlyProcessingOperationIndex) + && !(e.operation.equals(currentMergeOperation))) + .collect(Collectors.toList()); + for (OperationWithIndex operationWithIndex : operationsInMemoryInvolvingMergee) { SubmittedVariantOperationEntity operation = operationWithIndex.operation; List inactiveEntities = operation.getInactiveObjects().stream() - .map(entity -> replaceRSInSubmittedVariantInactiveEntity - (entity, prioritised.accessionToBeMerged, prioritised.accessionToKeep)) - .collect(Collectors.toList()); + .map(entity -> replaceRSInSubmittedVariantInactiveEntity + (entity, prioritised.accessionToBeMerged, prioritised.accessionToKeep)) + .collect(Collectors.toList()); SubmittedVariantOperationEntity updatedOperation = new SubmittedVariantOperationEntity(); updatedOperation.fill(operation.getEventType(), operation.getAccession(), operation.getReason(), @@ -377,11 +379,11 @@ private SubmittedVariantInactiveEntity replaceRSInSubmittedVariantInactiveEntity private void updateSplitCandidates(ClusteredVariantMergingPolicy.Priority prioritised) { Query queryForSplitCandidatesInvolvingTargetRS = query(getSplitCandidatesCriteria(this.assemblyAccession) - .and(RS_KEY_IN_OPERATIONS_COLLECTION) - .is(prioritised.accessionToKeep)); + .and(RS_KEY_IN_OPERATIONS_COLLECTION) + .is(prioritised.accessionToKeep)); Query queryForSplitCandidatesInvolvingMergee = query(getSplitCandidatesCriteria(this.assemblyAccession) - .and(RS_KEY_IN_OPERATIONS_COLLECTION) - .is(prioritised.accessionToBeMerged)); + .and(RS_KEY_IN_OPERATIONS_COLLECTION) + .is(prioritised.accessionToBeMerged)); // Since the mergee has been merged into the target RS, // the split candidates record for mergee are no longer valid - so, delete them! mongoTemplate.remove(queryForSplitCandidatesInvolvingMergee, SubmittedVariantOperationEntity.class); @@ -393,30 +395,30 @@ private void updateSplitCandidates(ClusteredVariantMergingPolicy.Priority priori List ssClusteredUnderTargetRS = this.submittedVariantAccessioningService .getByClusteredVariantAccessionIn(Collections.singletonList(prioritised.accessionToKeep), - ContigNamingConvention.NO_REPLACEMENT) + ContigNamingConvention.NO_REPLACEMENT) .stream() .filter(result -> result.getData().getReferenceSequenceAccession() - .equals(this.assemblyAccession)) + .equals(this.assemblyAccession)) .map(result -> new SubmittedVariantEntity(result.getAccession(), result.getHash(), - result.getData(), result.getVersion())) + result.getData(), result.getVersion())) .map(SubmittedVariantInactiveEntity::new) .collect(Collectors.toList()); Map> distinctLociInDBForTargetRS = ssClusteredUnderTargetRS.stream() - .map(SubmittedVariantInactiveEntity::toSubmittedVariantEntity) - .map(clusteringWriter::toClusteredVariantEntity) - .collect(Collectors.groupingBy(ClusteredVariantEntity::getHashedMessage)); + .map(SubmittedVariantInactiveEntity::toSubmittedVariantEntity) + .map(clusteringWriter::toClusteredVariantEntity) + .collect(Collectors.groupingBy(ClusteredVariantEntity::getHashedMessage)); //Update existing split candidates record for target RS if it exists. Else, create a new record! if (Objects.nonNull(splitCandidateInvolvingTargetRS) && distinctLociInDBForTargetRS.size() > 1) { mongoTemplate.updateFirst(query(where(ID_ATTRIBUTE).is(splitCandidateInvolvingTargetRS.getId())), - update(INACTIVE_OBJECT_ATTRIBUTE, ssClusteredUnderTargetRS), - SubmittedVariantOperationEntity.class); + update(INACTIVE_OBJECT_ATTRIBUTE, ssClusteredUnderTargetRS), + SubmittedVariantOperationEntity.class); } else { Set targetRSDistinctLoci = ssClusteredUnderTargetRS.stream().map(entity -> clusteringWriter - .getClusteredVariantHash(entity.getModel())) - .collect(Collectors.toSet()); + .getClusteredVariantHash(entity.getModel())) + .collect(Collectors.toSet()); // Condition for generating split operation: ensure that there is more than one locus sharing the target RS if (targetRSDistinctLoci.size() > 1) { // The new SPLIT candidate is for rs prioritised.accessionToKeep @@ -458,8 +460,8 @@ private void updateSubmittedVariants( List operations = svToUpdate.stream() - .map(sv -> buildSubmittedOperation(sv, prioritised.accessionToKeep)) - .collect(Collectors.toList()); + .map(sv -> buildSubmittedOperation(sv, prioritised.accessionToKeep)) + .collect(Collectors.toList()); mongoTemplate.insert(operations, submittedOperationCollection); metricCompute.addCount(ClusteringMetric.SUBMITTED_VARIANTS_UPDATE_OPERATIONS, operations.size()); } diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/io/RSSplitWriter.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/io/RSSplitWriter.java index 41b89f64e..21dbe5645 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/io/RSSplitWriter.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/io/RSSplitWriter.java @@ -16,22 +16,21 @@ package uk.ac.ebi.eva.accession.clustering.batch.io; import com.mongodb.MongoBulkWriteException; - import com.mongodb.client.result.UpdateResult; +import jakarta.annotation.Nonnull; import org.apache.commons.lang3.tuple.ImmutablePair; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.batch.core.JobExecution; +import org.springframework.batch.item.Chunk; import org.springframework.batch.item.ItemWriter; import org.springframework.data.mongodb.core.MongoTemplate; - import org.springframework.data.mongodb.core.query.Query; import org.springframework.data.mongodb.core.query.Update; import uk.ac.ebi.ampt2d.commons.accession.core.exceptions.AccessionCouldNotBeGeneratedException; import uk.ac.ebi.ampt2d.commons.accession.core.models.AccessionWrapper; import uk.ac.ebi.ampt2d.commons.accession.core.models.EventType; import uk.ac.ebi.ampt2d.commons.accession.persistence.mongodb.document.EventDocument; - import uk.ac.ebi.eva.accession.clustering.batch.io.ClusteredVariantSplittingPolicy.SplitDeterminants; import uk.ac.ebi.eva.accession.clustering.metric.ClusteringMetric; import uk.ac.ebi.eva.accession.core.model.IClusteredVariant; @@ -50,7 +49,6 @@ import uk.ac.ebi.eva.commons.core.models.contigalias.ContigNamingConvention; import uk.ac.ebi.eva.metrics.metric.MetricCompute; -import javax.annotation.Nonnull; import java.io.File; import java.io.FileWriter; import java.io.IOException; @@ -114,15 +112,14 @@ public RSSplitWriter(ClusteringWriter clusteringWriter, } @Override - public void write(@Nonnull List submittedVariantOperationEntities) + public void write(@Nonnull Chunk submittedVariantOperationEntities) throws MongoBulkWriteException, AccessionCouldNotBeGeneratedException, IOException { try { this.rsReportFileWriter = new FileWriter(this.rsReportFile, true); for (SubmittedVariantOperationEntity entity : submittedVariantOperationEntities) { writeRSSplit(entity); } - } - finally { + } finally { this.rsReportFileWriter.close(); } } @@ -159,17 +156,17 @@ public void writeRSSplit(SubmittedVariantOperationEntity submittedVariantOperati .entrySet().stream().map(rsHashAndAssociatedSS -> new SplitDeterminants( clusteringWriter.toClusteredVariantEntity(rsHashAndAssociatedSS.getValue() - .get(0)), + .get(0)), rsHashAndAssociatedSS.getKey(), rsHashAndAssociatedSS.getValue().size(), // Get lowest SS ID associated with a given RS hash rsHashAndAssociatedSS.getValue().stream() - .map(SubmittedVariantEntity::getAccession) - .min(Comparator.naturalOrder()).get())) + .map(SubmittedVariantEntity::getAccession) + .min(Comparator.naturalOrder()).get())) .collect(Collectors.toList()); // Based on the split policy, one of the hashes will retain the RS associated with it // and the other hashes should be associated with new RS IDs - List hashesThatShouldGetNewRS = getHashesThatShouldGetNewRS(splitCandidates); + List hashesThatShouldGetNewRS = getHashesThatShouldGetNewRS(splitCandidates); issueNewRSForHashes(hashesThatShouldGetNewRS, submittedVariantOperationEntity.getInactiveObjects()); } @@ -182,13 +179,13 @@ private void issueNewRSForHashes(List hashesThatShouldGetNewRS, throws AccessionCouldNotBeGeneratedException, IOException { Map> rsHashAndAssociatedSS = submittedVariantInactiveEntities - .stream() - .map(SubmittedVariantInactiveEntity::toSubmittedVariantEntity) - // If the split was processed multiple times due to multiple runs of the clustering pipeline - // ensure that the SS was not already assigned a new RS - .filter(ss -> !this.doesSSAlreadyHaveANewRS(ss)) - .collect(Collectors.groupingBy(this::getRSHashForSS)); - for (String rsHash: rsHashAndAssociatedSS.keySet()) { + .stream() + .map(SubmittedVariantInactiveEntity::toSubmittedVariantEntity) + // If the split was processed multiple times due to multiple runs of the clustering pipeline + // ensure that the SS was not already assigned a new RS + .filter(ss -> !this.doesSSAlreadyHaveANewRS(ss)) + .collect(Collectors.groupingBy(this::getRSHashForSS)); + for (String rsHash : rsHashAndAssociatedSS.keySet()) { if (hashesThatShouldGetNewRS.contains(rsHash)) { // Remove entry in clustered variant collections if hash already exists removeExistingHash(rsHash); @@ -197,19 +194,19 @@ private void issueNewRSForHashes(List hashesThatShouldGetNewRS, rsHashAndAssociatedSS.get(rsHash).get(0)); Long newRSAccession = this.clusteredVariantAccessioningService.getOrCreate( - Collections.singletonList(clusteredVariantEntity), jobExecution.getJobId().toString()) + Collections.singletonList(clusteredVariantEntity), jobExecution.getJobId().toString()) .get(0).getAccession(); ClusteringWriter.writeRSReportEntry(this.rsReportFileWriter, newRSAccession, rsHash); metricCompute.addCount(ClusteringMetric.CLUSTERED_VARIANTS_CREATED, 1); List associatedSSEntries = rsHashAndAssociatedSS.get(rsHash); - for (SubmittedVariantEntity submittedVariantEntity: associatedSSEntries) { - Long oldRSAccession = submittedVariantEntity.getClusteredVariantAccession(); + for (SubmittedVariantEntity submittedVariantEntity : associatedSSEntries) { + Long oldRSAccession = submittedVariantEntity.getClusteredVariantAccession(); logger.info("RS split operation: Associating ss{} with hash {} to newly issued rs{}...", - submittedVariantEntity.getAccession(), submittedVariantEntity.getHashedMessage(), - newRSAccession); + submittedVariantEntity.getAccession(), submittedVariantEntity.getHashedMessage(), + newRSAccession); associateNewRSToSS(newRSAccession, submittedVariantEntity); writeRSUpdateOperation(oldRSAccession, newRSAccession, - clusteringWriter.toClusteredVariantEntity(submittedVariantEntity)); + clusteringWriter.toClusteredVariantEntity(submittedVariantEntity)); writeSSUpdateOperation(oldRSAccession, newRSAccession, submittedVariantEntity); } } @@ -229,7 +226,7 @@ private boolean doesSSAlreadyHaveANewRS(SubmittedVariantEntity ssMarkedToReceive if (ssInDBOption.isPresent()) { ISubmittedVariant ssInDB = ssInDBOption.get(); return !(ssInDB.getClusteredVariantAccession() - .equals(ssMarkedToReceiveNewRS.getClusteredVariantAccession())); + .equals(ssMarkedToReceiveNewRS.getClusteredVariantAccession())); } return false; } @@ -270,12 +267,12 @@ private void writeRSUpdateOperation(Long oldRSAccession, Long newRSAccession, String splitOperationDescription = "Due to hash mismatch, rs" + newRSAccession + " was issued to split from rs" + oldRSAccession + "."; splitOperation.fill(EventType.RS_SPLIT, oldRSAccession, newRSAccession, splitOperationDescription, - Collections.singletonList(new ClusteredVariantInactiveEntity(clusteredVariantEntity))); + Collections.singletonList(new ClusteredVariantInactiveEntity(clusteredVariantEntity))); Query queryToCheckPreviousRSOperation = query(where(ACCESSION_ATTRIBUTE).is(oldRSAccession)) .addCriteria(where(EVENT_TYPE_ATTRIBUTE).is(EventType.RS_SPLIT)) .addCriteria(where(SPLIT_INTO_ATTRIBUTE).is(newRSAccession)) .addCriteria(where(ASSEMBLY_ATTRIBUTE_IN_CLUSTERED_OPERATIONS_COLLECTION) - .is(clusteredVariantEntity.getAssemblyAccession())); + .is(clusteredVariantEntity.getAssemblyAccession())); if (this.mongoTemplate.find(queryToCheckPreviousRSOperation, operationClass).isEmpty()) { this.mongoTemplate.insert(splitOperation, this.mongoTemplate.getCollectionName(operationClass)); metricCompute.addCount(ClusteringMetric.CLUSTERED_VARIANTS_RS_SPLIT, 1); @@ -292,13 +289,14 @@ private void writeSSUpdateOperation(Long oldRSAccession, Long newRSAccession, String updateOperationDescription = "SS was associated with the split RS rs" + newRSAccession + " that was split from rs" + oldRSAccession + " after remapping."; updateOperation.fill(EventType.UPDATED, submittedVariantEntity.getAccession(), updateOperationDescription, - Collections.singletonList(new SubmittedVariantInactiveEntity(submittedVariantEntity))); + Collections.singletonList(new SubmittedVariantInactiveEntity(submittedVariantEntity))); this.mongoTemplate.insert(updateOperation, this.mongoTemplate.getCollectionName(operationClass)); metricCompute.addCount(ClusteringMetric.SUBMITTED_VARIANTS_UPDATE_OPERATIONS, 1); } /** * Get hashes that should receive the new RS + * * @param splitCandidates List of triples * 1) ClusteredVariant object with the hash 2) the hash itself and * 3) the number of variants with that hash @@ -316,15 +314,15 @@ private List getHashesThatShouldGetNewRS(List hashThatShouldRetainOldRS.getClusteredVariantEntity().getAccession()); String hashToFind = hashThatShouldRetainOldRS.getRsHash(); Class otherCollection = - (rsCollectionToUse == DbsnpClusteredVariantEntity.class)? - ClusteredVariantEntity.class: DbsnpClusteredVariantEntity.class; + (rsCollectionToUse == DbsnpClusteredVariantEntity.class) ? + ClusteredVariantEntity.class : DbsnpClusteredVariantEntity.class; if (this.mongoTemplate.find(query(where(ID_ATTRIBUTE).is(hashToFind)), rsCollectionToUse).isEmpty()) { // Don't insert an entry with an RS hash if that hash already exists in another collection // This way, we at least make it possible for a clustering re-run to resolve a previously unresolved merge // See https://docs.google.com/spreadsheets/d/10NqzlcmKF5cVotJtnYhEFYvD_A7P9WsjN2p2ipYFKxE/edit#gid=0 // If we allow multiple hashes to co-exist in different collections, a re-run will be hindered // (see EVA-3132 and https://ebi-eva.slack.com/archives/C5A6MLDAR/p1676986532496859) - if (this.mongoTemplate.find(query(where(ID_ATTRIBUTE).is(hashToFind)),otherCollection).isEmpty()) { + if (this.mongoTemplate.find(query(where(ID_ATTRIBUTE).is(hashToFind)), otherCollection).isEmpty()) { this.mongoTemplate.insert(hashThatShouldRetainOldRS.getClusteredVariantEntity(), this.mongoTemplate.getCollectionName(rsCollectionToUse)); metricCompute.addCount(ClusteringMetric.CLUSTERED_VARIANTS_CREATED, 1); @@ -332,13 +330,13 @@ private List getHashesThatShouldGetNewRS(List logger.warn("Skipping creation of RS record " + hashThatShouldRetainOldRS.getClusteredVariantEntity() + " because the hash " + hashToFind + " is present in the " + otherCollection.getSimpleName() + " collection! This should NOT happen and likely indicates an RS merge that did not take place!" - +" See https://docs.google.com/spreadsheets/d/10NqzlcmKF5cVotJtnYhEFYvD_A7P9WsjN2p2ipYFKxE/edit#gid=0."); + + " See https://docs.google.com/spreadsheets/d/10NqzlcmKF5cVotJtnYhEFYvD_A7P9WsjN2p2ipYFKxE/edit#gid=0."); } } return splitCandidates.stream() - .map(SplitDeterminants::getRsHash) - .filter(rsHash -> !(rsHash.equals(hashThatShouldRetainOldRS.getRsHash()))) - .collect(Collectors.toList()); + .map(SplitDeterminants::getRsHash) + .filter(rsHash -> !(rsHash.equals(hashThatShouldRetainOldRS.getRsHash()))) + .collect(Collectors.toList()); } public void setJobExecution(JobExecution jobExecution) { diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/io/SSSplitWriter.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/io/SSSplitWriter.java index c8ce136f1..0eac74c2a 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/io/SSSplitWriter.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/io/SSSplitWriter.java @@ -16,20 +16,19 @@ package uk.ac.ebi.eva.accession.clustering.batch.io; import com.mongodb.MongoBulkWriteException; - +import jakarta.annotation.Nonnull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.batch.core.JobExecution; +import org.springframework.batch.item.Chunk; import org.springframework.batch.item.ItemWriter; import org.springframework.data.mongodb.core.MongoTemplate; - import org.springframework.data.mongodb.core.query.Query; import uk.ac.ebi.ampt2d.commons.accession.core.exceptions.AccessionCouldNotBeGeneratedException; import uk.ac.ebi.ampt2d.commons.accession.core.models.AccessionWrapper; import uk.ac.ebi.ampt2d.commons.accession.core.models.EventType; import uk.ac.ebi.ampt2d.commons.accession.core.models.GetOrCreateAccessionWrapper; import uk.ac.ebi.ampt2d.commons.accession.persistence.mongodb.document.EventDocument; - import uk.ac.ebi.eva.accession.clustering.batch.io.SubmittedVariantSplittingPolicy.SplitDeterminants; import uk.ac.ebi.eva.accession.clustering.metric.ClusteringMetric; import uk.ac.ebi.eva.accession.core.model.ISubmittedVariant; @@ -41,7 +40,6 @@ import uk.ac.ebi.eva.accession.core.service.nonhuman.SubmittedVariantAccessioningService; import uk.ac.ebi.eva.metrics.metric.MetricCompute; -import javax.annotation.Nonnull; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -95,11 +93,11 @@ public SSSplitWriter(String assembly, ClusteringWriter clusteringWriter, * Providing duplicates is harmless but may make the writer inefficient * since it will result in extra queries to the database. */ - public void write(@Nonnull List submittedVariantEntities) + public void write(@Nonnull Chunk submittedVariantEntities) throws MongoBulkWriteException, AccessionCouldNotBeGeneratedException, InstantiationException, IllegalAccessException { // We do this to avoid tedious generic type signatures in the method calls - List svesWithDuplicateID = new ArrayList<>(submittedVariantEntities); + List svesWithDuplicateID = new ArrayList<>(submittedVariantEntities.getItems()); // All the following steps are designed to be idempotent registerSplitCandidates(svesWithDuplicateID); @@ -110,19 +108,19 @@ public void write(@Nonnull List submittedVaria protected void processSplitCandidates(List splitCandidateOperations) throws AccessionCouldNotBeGeneratedException { Map svesToCreateWithNewIDs = new HashMap<>(); - for (SubmittedVariantOperationEntity operation: splitCandidateOperations) { + for (SubmittedVariantOperationEntity operation : splitCandidateOperations) { List splitCandidates = operation.getInactiveObjects() - .stream().map(svie -> new SplitDeterminants(svie.toSubmittedVariantEntity(), - svie.getHashedMessage(), - svie.getClusteredVariantAccession())) - .collect(Collectors.toList()); + .stream().map(svie -> new SplitDeterminants(svie.toSubmittedVariantEntity(), + svie.getHashedMessage(), + svie.getClusteredVariantAccession())) + .collect(Collectors.toList()); // Based on the split policy, one of the SS hashes will retain the SS associated with it // and the other hashes should be associated with new SS IDs - List svesThatShouldGetNewIDs = getSVEsThatShouldGetNewIDs(splitCandidates); + List svesThatShouldGetNewIDs = getSVEsThatShouldGetNewIDs(splitCandidates); - for(SplitDeterminants splitCandidate: splitCandidates) { - if(svesThatShouldGetNewIDs.contains(splitCandidate.getSubmittedVariantEntity())) { + for (SplitDeterminants splitCandidate : splitCandidates) { + if (svesThatShouldGetNewIDs.contains(splitCandidate.getSubmittedVariantEntity())) { svesToCreateWithNewIDs.put(splitCandidate.getSSHash(), splitCandidate.getSubmittedVariantEntity()); } } @@ -141,14 +139,13 @@ private void excludeSSWithAlreadyUpdatedIDs(Map List> existingSSList = this.submittedVariantAccessioningService.get(new ArrayList<>(ssHashesAndAssociatedSS.values())); - for(AccessionWrapper existingSSEntryInDB: existingSSList) { + for (AccessionWrapper existingSSEntryInDB : existingSSList) { // If some existing SS in the database have already been provided an updated accession // when processing a split operation previously, remove such SS from being considered for split String existingSSEntryInDBHash = existingSSEntryInDB.getHash(); - if(ssHashesAndAssociatedSS.containsKey(existingSSEntryInDBHash) && + if (ssHashesAndAssociatedSS.containsKey(existingSSEntryInDBHash) && !existingSSEntryInDB.getAccession().equals(ssHashesAndAssociatedSS - .get(existingSSEntryInDBHash).getAccession())) - { + .get(existingSSEntryInDBHash).getAccession())) { ssHashesAndAssociatedSS.remove(existingSSEntryInDB.getHash()); } } @@ -168,34 +165,33 @@ protected void removeCurrentSSEntriesInDBForSplitCandidates(Set ssHashes Map> dbsnpSvoeOpsToWrite = new HashMap<>(); - for(GetOrCreateAccessionWrapper newlyCreatedSVE: newlyCreatedSVEs) { + for (GetOrCreateAccessionWrapper newlyCreatedSVE : newlyCreatedSVEs) { SubmittedVariantEntity oldSVE = oldSVEs.get(newlyCreatedSVE.getHash()); String idForSplitOperation = String.format("SS_SPLIT_FROM_%s_TO_%s", oldSVE.getAccession(), - newlyCreatedSVE.getAccession()); + newlyCreatedSVE.getAccession()); logger.info("Processed split operation " + idForSplitOperation + "..."); SubmittedVariantOperationEntity splitOperation = new SubmittedVariantOperationEntity(); splitOperation.fill(EventType.SS_SPLIT, - oldSVE.getAccession(), newlyCreatedSVE.getAccession(), - String.format("SS split from %s to %s", oldSVE.getAccession(), - newlyCreatedSVE.getAccession()) + - " due to same SS ID being assigned to different SS loci", - Collections.singletonList(new SubmittedVariantInactiveEntity(oldSVE))); + oldSVE.getAccession(), newlyCreatedSVE.getAccession(), + String.format("SS split from %s to %s", oldSVE.getAccession(), + newlyCreatedSVE.getAccession()) + + " due to same SS ID being assigned to different SS loci", + Collections.singletonList(new SubmittedVariantInactiveEntity(oldSVE))); splitOperation.setId(idForSplitOperation); if (this.clusteringWriter.isEvaSubmittedVariant(oldSVE)) { svoeOpsToWrite.put(idForSplitOperation, splitOperation); - } - else { + } else { dbsnpSvoeOpsToWrite.put(idForSplitOperation, splitOperation); } } // Remove entries for which operations have already been recorded this.mongoTemplate.find(query(where(ID_ATTRIBUTE).in(svoeOpsToWrite.keySet())), - SubmittedVariantOperationEntity.class) - .forEach(existingSvoeOp -> svoeOpsToWrite.remove(existingSvoeOp.getId())); + SubmittedVariantOperationEntity.class) + .forEach(existingSvoeOp -> svoeOpsToWrite.remove(existingSvoeOp.getId())); this.mongoTemplate.find(query(where(ID_ATTRIBUTE).in(dbsnpSvoeOpsToWrite.keySet())), - DbsnpSubmittedVariantOperationEntity.class) - .forEach(existingDbsnpSvoeOp -> dbsnpSvoeOpsToWrite.remove(existingDbsnpSvoeOp.getId())); + DbsnpSubmittedVariantOperationEntity.class) + .forEach(existingDbsnpSvoeOp -> dbsnpSvoeOpsToWrite.remove(existingDbsnpSvoeOp.getId())); if (svoeOpsToWrite.values().size() > 0) { this.mongoTemplate.insert(svoeOpsToWrite.values(), SubmittedVariantOperationEntity.class); @@ -209,7 +205,7 @@ protected void removeSplitCandidates(List split List splitCandidateIDsToFind = splitCandidatesForCurrentBatch.stream().map( SubmittedVariantOperationEntity::getId).collect(Collectors.toList()); this.mongoTemplate.remove(query(where(ID_ATTRIBUTE).in(splitCandidateIDsToFind)), - SubmittedVariantOperationEntity.class); + SubmittedVariantOperationEntity.class); } protected void registerSplitCandidates(List svesWithDuplicateID) { @@ -225,7 +221,7 @@ protected void registerSplitCandidates(List svesWithDupl this.allSplitCandidatesForCurrentBatch.addAll(existingSplitCandidateOperationsInDB); Set existingSplitCandidateOperationIdsInDB = existingSplitCandidateOperationsInDB.stream().map(SubmittedVariantOperationEntity::getId) - .collect(Collectors.toSet()); + .collect(Collectors.toSet()); // Remove operations that are already in DB idsForSplitCandidateOperationsToWrite.removeAll(existingSplitCandidateOperationIdsInDB); // Bulk write split candidates for current batch @@ -236,7 +232,7 @@ private Map> getIdSVEMapForSplitCandidateOp List svesWithDuplicateID) { Map> idSVEMapForSplitCandidateOperations = new HashMap<>(); List duplicateSSIDs = svesWithDuplicateID.stream().map(SubmittedVariantEntity::getAccession) - .collect(Collectors.toList()); + .collect(Collectors.toList()); // The input list that we get may only have one of the SS entries that share duplicate IDs // We have to fetch the actual duplicates themselves in order to construct split candidates @@ -245,20 +241,19 @@ private Map> getIdSVEMapForSplitCandidateOp .getAllActiveByAssemblyAndAccessionIn(this.assembly, duplicateSSIDs) .stream() .map(result -> new SubmittedVariantEntity(result.getAccession(), result.getHash(), - result.getData(), result.getVersion())) + result.getData(), result.getVersion())) .filter(sve -> !this.variantHasMultiMapOrMismatchedAlleles(sve) && Objects.isNull(sve.getRemappedFrom())) .collect(Collectors.toList()); - for (SubmittedVariantEntity sve: svesWithDuplicateIDAlongWithActualDuplicates) { + for (SubmittedVariantEntity sve : svesWithDuplicateIDAlongWithActualDuplicates) { String idForSplitCandidateOperation = String.format("SS_SPLIT_CANDIDATES_%s_%s", - sve.getReferenceSequenceAccession(), - sve.getAccession()); + sve.getReferenceSequenceAccession(), + sve.getAccession()); if (!idSVEMapForSplitCandidateOperations.containsKey(idForSplitCandidateOperation)) { idSVEMapForSplitCandidateOperations.put(idForSplitCandidateOperation, - new ArrayList<>(Collections.singletonList(sve))); - } - else { + new ArrayList<>(Collections.singletonList(sve))); + } else { idSVEMapForSplitCandidateOperations.get(idForSplitCandidateOperation).add(sve); } } @@ -269,7 +264,7 @@ private void writeSplitCandidatesForCurrentBatchToDB(Map idsForSplitCandidateOperationsToWrite) { // Only new split candidates in the current batch should be written to the database List newSplitCandidatesForCurrentBatch = new ArrayList<>(); - for (String id: idsForSplitCandidateOperationsToWrite) { + for (String id : idsForSplitCandidateOperationsToWrite) { SubmittedVariantOperationEntity splitCandidateOperation = new SubmittedVariantOperationEntity(); List svesThatShareSameID = idSVEMap.get(id); // Ensure that non-duplicate SS IDs are not processed @@ -278,9 +273,9 @@ private void writeSplitCandidatesForCurrentBatchToDB(Map> { diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/io/qc/DuplicateRSAccQCWriter.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/io/qc/DuplicateRSAccQCWriter.java index 3a8b32026..321929bcd 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/io/qc/DuplicateRSAccQCWriter.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/io/qc/DuplicateRSAccQCWriter.java @@ -1,9 +1,10 @@ package uk.ac.ebi.eva.accession.clustering.batch.io.qc; -import gherkin.deps.com.google.gson.Gson; -import gherkin.deps.com.google.gson.GsonBuilder; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.batch.item.Chunk; import org.springframework.batch.item.ExecutionContext; import org.springframework.batch.item.ItemStreamException; import org.springframework.batch.item.ItemStreamWriter; @@ -48,7 +49,7 @@ public void close() throws ItemStreamException { } @Override - public void write(List> listOfDuplicateRSAccQCResultLists) throws Exception { + public void write(Chunk> listOfDuplicateRSAccQCResultLists) throws Exception { for (List duplicateRSAccQCResultList : listOfDuplicateRSAccQCResultLists) { if (duplicateRSAccQCResultList != null && !duplicateRSAccQCResultList.isEmpty()) { appendToFile(duplicateRSAccQCResultList); diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/io/qc/ExtraneousRSReporter.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/io/qc/ExtraneousRSReporter.java index 5b78627bd..9a0caf252 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/io/qc/ExtraneousRSReporter.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/io/qc/ExtraneousRSReporter.java @@ -16,9 +16,10 @@ package uk.ac.ebi.eva.accession.clustering.batch.io.qc; import com.mongodb.MongoBulkWriteException; -import org.apache.commons.collections.CollectionUtils; +import jakarta.annotation.Nonnull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.batch.item.Chunk; import org.springframework.batch.item.ItemWriter; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.data.mongodb.core.query.Criteria; @@ -27,9 +28,8 @@ import uk.ac.ebi.eva.accession.clustering.batch.io.qc.QCMongoCollections.qcRSIdInSS; import uk.ac.ebi.eva.accession.core.model.eva.ClusteredVariantEntity; -import javax.annotation.Nonnull; -import java.util.Arrays; import java.util.List; +import java.util.Set; import java.util.stream.Collectors; import static org.springframework.data.mongodb.core.query.Criteria.where; @@ -50,24 +50,24 @@ public ExtraneousRSReporter(String assemblyAccession, MongoTemplate mongoTemplat } @Override - public void write(@Nonnull List clusteredVariantEntities) + public void write(@Nonnull Chunk clusteredVariantEntities) throws MongoBulkWriteException, AccessionCouldNotBeGeneratedException { reportExtraneousRS(clusteredVariantEntities); } - private void reportExtraneousRS(List clusteredVariantEntities) { + private void reportExtraneousRS(Chunk clusteredVariantEntities) { String assemblyAccessionPrefix = QCMongoCollections.getAssemblyAccessionPrefix(this.assemblyAccession); - List idsFromRSIDCollection = clusteredVariantEntities.stream().map( + List idsFromRSIDCollection = clusteredVariantEntities.getItems().stream().map( entity -> assemblyAccessionPrefix + entity.getAccession()).distinct().collect(Collectors.toList()); - Criteria[] criteriaToLookupIDs = + Criteria[] criteriaToLookupIDs = idsFromRSIDCollection.stream().map(id -> where(IDAttribute).is(id)).toArray(Criteria[]::new); if (criteriaToLookupIDs.length == 0) return; Query queryToLookupIDs = new Query(new Criteria().orOperator(criteriaToLookupIDs)); - List idsInSSIDCollection = this.mongoTemplate.find(queryToLookupIDs, qcRSIdInSS.class) - .stream().map(qcRSIdInSS::getId).collect(Collectors.toList()); + Set idsInSSIDCollection = this.mongoTemplate.find(queryToLookupIDs, qcRSIdInSS.class) + .stream().map(qcRSIdInSS::getId).collect(Collectors.toSet()); - Arrays.stream(CollectionUtils.subtract(idsFromRSIDCollection, idsInSSIDCollection).toArray()) - .map(Object::toString) + idsFromRSIDCollection.stream() + .filter(id -> !idsInSSIDCollection.contains(id)) .forEach(extraneousRS -> logger.error("RS ID rs{} was not assigned to any SS in the assembly {}", extraneousRS.replace(assemblyAccessionPrefix, ""), this.assemblyAccession)); diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/io/qc/MissingCveReporter.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/io/qc/MissingCveReporter.java index 470902a6a..4d1ea4e64 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/io/qc/MissingCveReporter.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/io/qc/MissingCveReporter.java @@ -15,16 +15,16 @@ */ package uk.ac.ebi.eva.accession.clustering.batch.io.qc; +import jakarta.annotation.Nonnull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.batch.item.Chunk; import org.springframework.batch.item.ItemWriter; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.data.mongodb.core.query.Query; - import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpClusteredVariantEntity; import uk.ac.ebi.eva.accession.core.model.eva.ClusteredVariantEntity; -import javax.annotation.Nonnull; import java.util.List; import java.util.Map; import java.util.Objects; @@ -48,9 +48,9 @@ public MissingCveReporter(MongoTemplate mongoTemplate) { } @Override - public void write(@Nonnull List rsHashPairs) { - Map hashToRs = rsHashPairs.stream() - .collect(Collectors.toMap(RSHashPair::getHash, RSHashPair::getRsId)); + public void write(@Nonnull Chunk rsHashPairs) { + Map hashToRs = rsHashPairs.getItems().stream() + .collect(Collectors.toMap(RSHashPair::getHash, RSHashPair::getRsId)); Map results = findClusteredVariantsInDb(hashToRs); for (String hash : hashToRs.keySet()) { @@ -71,7 +71,7 @@ private Map findClusteredVariantsInDb(Map evaResults = mongoTemplate.find(query, ClusteredVariantEntity.class); List dbsnpResults = mongoTemplate.find(query, DbsnpClusteredVariantEntity.class); return Stream.concat(evaResults.stream(), dbsnpResults.stream()) - .collect(Collectors.toMap(ClusteredVariantEntity::getHashedMessage, Function.identity())); + .collect(Collectors.toMap(ClusteredVariantEntity::getHashedMessage, Function.identity())); } } diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/io/qc/PendingMergeSplitReporter.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/io/qc/PendingMergeSplitReporter.java index 628adf5c6..4cb1cc843 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/io/qc/PendingMergeSplitReporter.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/io/qc/PendingMergeSplitReporter.java @@ -16,21 +16,23 @@ package uk.ac.ebi.eva.accession.clustering.batch.io.qc; import com.mongodb.MongoBulkWriteException; +import jakarta.annotation.Nonnull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.batch.item.Chunk; import org.springframework.batch.item.ItemWriter; import org.springframework.dao.DuplicateKeyException; import org.springframework.data.mongodb.core.BulkOperations; import org.springframework.data.mongodb.core.MongoTemplate; import uk.ac.ebi.ampt2d.commons.accession.core.exceptions.AccessionCouldNotBeGeneratedException; - import uk.ac.ebi.eva.accession.clustering.batch.io.ClusteringWriter; import uk.ac.ebi.eva.accession.clustering.batch.io.qc.QCMongoCollections.qcRSHashInSS; import uk.ac.ebi.eva.accession.clustering.batch.io.qc.QCMongoCollections.qcRSIdInSS; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; -import javax.annotation.Nonnull; -import java.util.*; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; import java.util.stream.Collectors; import static org.springframework.data.mongodb.core.query.Criteria.where; @@ -57,18 +59,18 @@ public PendingMergeSplitReporter(String assemblyAccession, ClusteringWriter clus } @Override - public void write(@Nonnull List submittedVariantEntities) + public void write(@Nonnull Chunk submittedVariantEntities) throws MongoBulkWriteException, AccessionCouldNotBeGeneratedException { if (submittedVariantEntities.size() > 0) { reportPendingSplitsAndMerges(submittedVariantEntities); } } - private void reportPendingSplitsAndMerges(List submittedVariantEntities) { + private void reportPendingSplitsAndMerges(Chunk submittedVariantEntities) { Map hashAndAssociatedRS = new HashMap<>(); Map rsAndAssociatedHash = new HashMap<>(); String assemblyAccessionPrefix = QCMongoCollections.getAssemblyAccessionPrefix(this.assemblyAccession); - for (SubmittedVariantEntity submittedVariantEntity :submittedVariantEntities) { + for (SubmittedVariantEntity submittedVariantEntity : submittedVariantEntities) { Long rsID = submittedVariantEntity.getClusteredVariantAccession(); if (Objects.nonNull(rsID)) { String rsHash = clusteringWriter.toClusteredVariantEntity(submittedVariantEntity).getHashedMessage(); @@ -90,33 +92,32 @@ private void reportPendingSplitsAndMerges(List if (hashAndAssociatedRS.size() > 0) { bulkHashInsert.execute(); } - } - catch (DuplicateKeyException duplicateKeyException) { + } catch (DuplicateKeyException duplicateKeyException) { MongoBulkWriteException writeException = ((MongoBulkWriteException) duplicateKeyException.getCause()); for (String hashWithAssemblyPrefix : extractUniqueHashesForDuplicateKeyError(writeException).collect( Collectors.toList())) { qcRSHashInSS result = this.mongoTemplate.findOne(query(where(IDAttribute).is(hashWithAssemblyPrefix)), - qcRSHashInSS.class); - reportMultipleRSWithSameHash (hashAndAssociatedRS, assemblyAccessionPrefix, result.getRsID(), - hashWithAssemblyPrefix.replace(assemblyAccessionPrefix, "")); - }; + qcRSHashInSS.class); + reportMultipleRSWithSameHash(hashAndAssociatedRS, assemblyAccessionPrefix, result.getRsID(), + hashWithAssemblyPrefix.replace(assemblyAccessionPrefix, "")); + } + ; } try { if (rsAndAssociatedHash.size() > 0) { bulkIDInsert.execute(); } - } - catch (DuplicateKeyException duplicateKeyException) { + } catch (DuplicateKeyException duplicateKeyException) { MongoBulkWriteException writeException = ((MongoBulkWriteException) duplicateKeyException.getCause()); for (String idWithAssemblyPrefix : extractUniqueHashesForDuplicateKeyError(writeException).collect( Collectors.toList())) { qcRSIdInSS result = this.mongoTemplate.findOne(query(where(IDAttribute).is(idWithAssemblyPrefix)), - qcRSIdInSS.class); + qcRSIdInSS.class); reportSameRSWithMultipleHashes(rsAndAssociatedHash, assemblyAccessionPrefix, - Long.parseLong(idWithAssemblyPrefix.replace(assemblyAccessionPrefix, "")), - result.getHash()); - }; + Long.parseLong(idWithAssemblyPrefix.replace(assemblyAccessionPrefix, "")), + result.getHash()); + } } } diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/io/qc/RSReader.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/io/qc/RSReader.java index c1eb7e253..2576aee8a 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/io/qc/RSReader.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/io/qc/RSReader.java @@ -19,6 +19,7 @@ import com.mongodb.client.FindIterable; import com.mongodb.client.MongoCursor; import com.mongodb.client.model.Filters; +import jakarta.annotation.Nonnull; import org.bson.Document; import org.bson.conversions.Bson; import org.slf4j.Logger; @@ -31,8 +32,6 @@ import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpClusteredVariantEntity; import uk.ac.ebi.eva.accession.core.model.eva.ClusteredVariantEntity; -import javax.annotation.Nonnull; - public class RSReader implements ItemStreamReader { @@ -79,7 +78,7 @@ public void open(@Nonnull ExecutionContext executionContext) throws ItemStreamEx public void initializeReader() { Bson query = Filters.and(Filters.in(ASSEMBLY_FIELD, assembly), - Filters.not(Filters.exists(MAPPING_WEIGHT_FIELD))); + Filters.not(Filters.exists(MAPPING_WEIGHT_FIELD))); logger.info("Issuing find: {}", query); FindIterable clusteredVariantsDbsnp = @@ -94,9 +93,9 @@ public void initializeReader() { private FindIterable getClusteredVariants(Bson query, Class entityClass) { return mongoTemplate.getCollection(mongoTemplate.getCollectionName(entityClass)) - .find(query) - .noCursorTimeout(true) - .batchSize(chunkSize); + .find(query) + .noCursorTimeout(true) + .batchSize(chunkSize); } @Override diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/io/qc/SSReader.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/io/qc/SSReader.java index fda379834..eb4305014 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/io/qc/SSReader.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/io/qc/SSReader.java @@ -19,6 +19,7 @@ import com.mongodb.client.FindIterable; import com.mongodb.client.MongoCursor; import com.mongodb.client.model.Filters; +import jakarta.annotation.Nonnull; import org.bson.Document; import org.bson.conversions.Bson; import org.slf4j.Logger; @@ -28,12 +29,9 @@ import org.springframework.batch.item.ItemStreamReader; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.data.mongodb.core.convert.MongoConverter; - import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpSubmittedVariantEntity; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; -import javax.annotation.Nonnull; - public class SSReader implements ItemStreamReader { @@ -80,7 +78,7 @@ public void open(ExecutionContext executionContext) throws ItemStreamException { public void initializeReader() { Bson query = Filters.and(Filters.in(ASSEMBLY_FIELD, assembly), - Filters.not(Filters.exists(MAPPING_WEIGHT_FIELD))); + Filters.not(Filters.exists(MAPPING_WEIGHT_FIELD))); logger.info("Issuing find: {}", query); FindIterable submittedVariantsDbsnp = @@ -95,9 +93,9 @@ public void initializeReader() { private FindIterable getSubmittedVariants(Bson query, Class entityClass) { return mongoTemplate.getCollection(mongoTemplate.getCollectionName(entityClass)) - .find(query) - .noCursorTimeout(true) - .batchSize(chunkSize); + .find(query) + .noCursorTimeout(true) + .batchSize(chunkSize); } @Override diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/processors/VariantToSubmittedVariantEntityProcessor.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/processors/VariantToSubmittedVariantEntityProcessor.java index 5f0032d07..ef1b7c376 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/processors/VariantToSubmittedVariantEntityProcessor.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/processors/VariantToSubmittedVariantEntityProcessor.java @@ -18,15 +18,14 @@ import org.springframework.batch.item.ItemProcessor; import org.springframework.util.Assert; import uk.ac.ebi.ampt2d.commons.accession.hashing.SHA1HashingFunction; - import uk.ac.ebi.eva.accession.core.model.ISubmittedVariant; import uk.ac.ebi.eva.accession.core.model.SubmittedVariant; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; import uk.ac.ebi.eva.accession.core.summary.SubmittedVariantSummaryFunction; -import uk.ac.ebi.eva.remapping.source.batch.io.VariantContextWriter; -import uk.ac.ebi.eva.remapping.source.batch.processors.SubmittedVariantToVariantContextProcessor; import uk.ac.ebi.eva.commons.core.models.pipeline.Variant; import uk.ac.ebi.eva.commons.core.models.pipeline.VariantSourceEntry; +import uk.ac.ebi.eva.remapping.source.batch.io.VariantContextWriter; +import uk.ac.ebi.eva.remapping.source.batch.processors.SubmittedVariantToVariantContextProcessor; import java.util.function.Function; import java.util.stream.Collectors; diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/RunnerConfiguration.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/RunnerConfiguration.java index d17811c9f..f3f598c7b 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/RunnerConfiguration.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/RunnerConfiguration.java @@ -16,15 +16,10 @@ */ package uk.ac.ebi.eva.accession.clustering.configuration.batch; -import org.springframework.batch.core.configuration.annotation.BatchConfigurer; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import uk.ac.ebi.eva.commons.batch.configuration.SpringBoot1CompatibilityConfiguration; import uk.ac.ebi.eva.commons.batch.job.JobExecutionApplicationListener; -import javax.persistence.EntityManagerFactory; -import javax.sql.DataSource; - @Configuration public class RunnerConfiguration { @@ -32,11 +27,4 @@ public class RunnerConfiguration { public JobExecutionApplicationListener jobExecutionApplicationListener() { return new JobExecutionApplicationListener(); } - - @Bean - public BatchConfigurer configurer(DataSource dataSource, EntityManagerFactory entityManagerFactory) - throws Exception { - return SpringBoot1CompatibilityConfiguration.getSpringBoot1CompatibleBatchConfigurer(dataSource, - entityManagerFactory); - } } diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/io/RSMergeAndSplitCandidatesReaderConfiguration.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/io/RSMergeAndSplitCandidatesReaderConfiguration.java index d009ee020..6b0ef84ee 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/io/RSMergeAndSplitCandidatesReaderConfiguration.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/io/RSMergeAndSplitCandidatesReaderConfiguration.java @@ -15,7 +15,7 @@ */ package uk.ac.ebi.eva.accession.clustering.configuration.batch.io; -import com.mongodb.DBCollection; +import org.springframework.batch.item.Chunk; import org.springframework.batch.item.ItemWriter; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -24,7 +24,6 @@ import org.springframework.data.mongodb.core.query.Criteria; import org.springframework.data.mongodb.core.query.Meta; import org.springframework.data.mongodb.core.query.Query; - import uk.ac.ebi.ampt2d.commons.accession.core.models.EventType; import uk.ac.ebi.eva.accession.clustering.configuration.InputParametersConfiguration; import uk.ac.ebi.eva.accession.clustering.parameters.InputParameters; @@ -33,7 +32,6 @@ import uk.ac.ebi.eva.commons.mongodb.readers.MongoDbCursorItemReader; import java.util.Arrays; -import java.util.List; import static org.springframework.data.mongodb.core.query.Criteria.where; import static org.springframework.data.mongodb.core.query.Query.query; @@ -60,19 +58,18 @@ public class RSMergeAndSplitCandidatesReaderConfiguration { public static final String SPLIT_CANDIDATE_ID_PREFIX = "RSSC"; public static Criteria getSplitCandidatesCriteria(String assemblyAccession) { - return where(DBCollection.ID_FIELD_NAME).regex("^" + SPLIT_CANDIDATE_ID_PREFIX + "_" - + assemblyAccession + "_.*"); + return where("_id").regex("^" + SPLIT_CANDIDATE_ID_PREFIX + "_" + + assemblyAccession + "_.*"); } public static Criteria getMergeCandidatesCriteria(String assemblyAccession) { - return where(DBCollection.ID_FIELD_NAME).regex("^" + MERGE_CANDIDATE_ID_PREFIX + "_" - + assemblyAccession + "_.*"); + return where("_id").regex("^" + MERGE_CANDIDATE_ID_PREFIX + "_" + + assemblyAccession + "_.*"); } @Bean(RS_SPLIT_CANDIDATES_READER) public MongoDbCursorItemReader rsSplitCandidatesReader(MongoTemplate mongoTemplate, - InputParameters parameters) - { + InputParameters parameters) { MongoDbCursorItemReader mongoItemReader = new MongoDbCursorItemReader<>(); mongoItemReader.setMongoTemplate(mongoTemplate); mongoItemReader.setTargetType(SubmittedVariantOperationEntity.class); @@ -90,8 +87,7 @@ public MongoDbCursorItemReader rsSplitCandidate @Bean(RS_MERGE_CANDIDATES_READER) public MongoDbCursorItemReader rsMergeCandidatesReader(MongoTemplate mongoTemplate, - InputParameters parameters) - { + InputParameters parameters) { MongoDbCursorItemReader mongoItemReader = new MongoDbCursorItemReader<>(); mongoItemReader.setMongoTemplate(mongoTemplate); mongoItemReader.setTargetType(SubmittedVariantOperationEntity.class); @@ -116,18 +112,20 @@ public MongoDbCursorItemReader rsMergeCandidate public static class NoOpItemWriter implements ItemWriter { private final MongoTemplate mongoTemplate; private final InputParameters parameters; + public NoOpItemWriter(MongoTemplate mongoTemplate, InputParameters parameters) { this.mongoTemplate = mongoTemplate; this.parameters = parameters; } + @Override - public void write(List items) throws Exception { + public void write(Chunk items) throws Exception { Query queryToRemoveMergeAndSplitCandidates = query(where(ASSEMBLY_FIELD).is(parameters.getAssemblyAccession())) - .addCriteria(where(EVENT_TYPE_FIELD).in( - Arrays.asList(MERGE_CANDIDATES_EVENT_TYPE.toString(), - SPLIT_CANDIDATES_EVENT_TYPE.toString())) - ); + .addCriteria(where(EVENT_TYPE_FIELD).in( + Arrays.asList(MERGE_CANDIDATES_EVENT_TYPE.toString(), + SPLIT_CANDIDATES_EVENT_TYPE.toString())) + ); mongoTemplate.remove(queryToRemoveMergeAndSplitCandidates, SUBMITTED_VARIANT_OPERATIONS_COLLECTION); } } diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/BackPropagateRSJobConfiguration.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/BackPropagateRSJobConfiguration.java index 3f0c1551e..ec9f999b8 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/BackPropagateRSJobConfiguration.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/BackPropagateRSJobConfiguration.java @@ -18,13 +18,17 @@ import org.springframework.batch.core.Job; import org.springframework.batch.core.Step; import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; -import org.springframework.batch.core.configuration.annotation.JobBuilderFactory; +import org.springframework.batch.core.job.builder.JobBuilder; import org.springframework.batch.core.launch.support.RunIdIncrementer; +import org.springframework.batch.core.repository.JobRepository; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.*; +import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.BACK_PROPAGATE_NEW_RS_JOB; +import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.BACK_PROPAGATE_NEW_RS_STEP; +import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.BACK_PROPAGATE_SPLIT_OR_MERGED_RS_JOB; +import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.BACK_PROPAGATE_SPLIT_OR_MERGED_RS_STEP; @Configuration @EnableBatchProcessing @@ -34,8 +38,8 @@ public class BackPropagateRSJobConfiguration { public Job backPropagateNewRSJob( // Back-propagate RS that were newly created in the remapped assembly @Qualifier(BACK_PROPAGATE_NEW_RS_STEP) Step backPropagateNewRSStep, - JobBuilderFactory jobBuilderFactory) { - return jobBuilderFactory.get(BACK_PROPAGATE_NEW_RS_JOB) + JobRepository jobRepository) { + return new JobBuilder(BACK_PROPAGATE_NEW_RS_JOB, jobRepository) .incrementer(new RunIdIncrementer()) .start(backPropagateNewRSStep) .build(); @@ -47,8 +51,8 @@ public Job backPropagateNewRSJob( public Job backPropagateSplitOrMergedRSJob( // Back-propagate RS in the remapped assembly that were split or merged @Qualifier(BACK_PROPAGATE_SPLIT_OR_MERGED_RS_STEP) Step backPropagateSplitMergedRSStep, - JobBuilderFactory jobBuilderFactory) { - return jobBuilderFactory.get(BACK_PROPAGATE_SPLIT_OR_MERGED_RS_JOB) + JobRepository jobRepository) { + return new JobBuilder(BACK_PROPAGATE_SPLIT_OR_MERGED_RS_JOB, jobRepository) .incrementer(new RunIdIncrementer()) .start(backPropagateSplitMergedRSStep) .build(); diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/ClusterUnclusteredVariantsJobConfiguration.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/ClusterUnclusteredVariantsJobConfiguration.java index 2399d090c..3740f1fd2 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/ClusterUnclusteredVariantsJobConfiguration.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/ClusterUnclusteredVariantsJobConfiguration.java @@ -19,8 +19,9 @@ import org.springframework.batch.core.JobExecutionListener; import org.springframework.batch.core.Step; import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; -import org.springframework.batch.core.configuration.annotation.JobBuilderFactory; +import org.springframework.batch.core.job.builder.JobBuilder; import org.springframework.batch.core.launch.support.RunIdIncrementer; +import org.springframework.batch.core.repository.JobRepository; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -47,15 +48,15 @@ public Job clusteringFromMongoJob( @Qualifier(CLUSTERING_NON_CLUSTERED_VARIANTS_FROM_MONGO_STEP) Step clusteringNonClusteredVariantsFromMongoStep, @Qualifier(ACCESSIONING_SHUTDOWN_STEP) Step accessioningShutdownStep, @Qualifier(JOB_EXECUTION_LISTENER) JobExecutionListener jobExecutionListener, - JobBuilderFactory jobBuilderFactory) { - return jobBuilderFactory.get(CLUSTER_UNCLUSTERED_VARIANTS_JOB) - .incrementer(new RunIdIncrementer()) - .start(processRSMergeCandidatesStep) - .next(processRSSplitCandidatesStep) - .next(clearRSMergeAndSplitCandidatesStep) - .next(clusteringNonClusteredVariantsFromMongoStep) - .next(accessioningShutdownStep) - .listener(jobExecutionListener) - .build(); + JobRepository jobRepository) { + return new JobBuilder(CLUSTER_UNCLUSTERED_VARIANTS_JOB, jobRepository) + .incrementer(new RunIdIncrementer()) + .start(processRSMergeCandidatesStep) + .next(processRSSplitCandidatesStep) + .next(clearRSMergeAndSplitCandidatesStep) + .next(clusteringNonClusteredVariantsFromMongoStep) + .next(accessioningShutdownStep) + .listener(jobExecutionListener) + .build(); } } diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/ClusteringFromMongoJobConfiguration.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/ClusteringFromMongoJobConfiguration.java index ea1c4a653..e4c27393d 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/ClusteringFromMongoJobConfiguration.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/ClusteringFromMongoJobConfiguration.java @@ -20,25 +20,22 @@ import org.springframework.batch.core.JobExecution; import org.springframework.batch.core.JobExecutionListener; import org.springframework.batch.core.Step; -import org.springframework.batch.core.StepContribution; import org.springframework.batch.core.StepExecution; - import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; -import org.springframework.batch.core.configuration.annotation.JobBuilderFactory; -import org.springframework.batch.core.configuration.annotation.StepBuilderFactory; import org.springframework.batch.core.job.builder.FlowBuilder; +import org.springframework.batch.core.job.builder.JobBuilder; import org.springframework.batch.core.job.flow.FlowExecutionStatus; import org.springframework.batch.core.job.flow.JobExecutionDecider; import org.springframework.batch.core.job.flow.support.SimpleFlow; import org.springframework.batch.core.launch.support.RunIdIncrementer; -import org.springframework.batch.core.scope.context.ChunkContext; -import org.springframework.batch.core.step.tasklet.Tasklet; +import org.springframework.batch.core.repository.JobRepository; +import org.springframework.batch.core.step.builder.StepBuilder; import org.springframework.batch.repeat.RepeatStatus; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.lang.NonNull; - +import org.springframework.transaction.PlatformTransactionManager; import uk.ac.ebi.eva.accession.clustering.parameters.InputParameters; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.ACCESSIONING_SHUTDOWN_STEP; @@ -60,20 +57,17 @@ private JobExecutionDecider isRemappedAssemblyPresent(InputParameters inputParam @Override @NonNull public FlowExecutionStatus decide(@NonNull JobExecution jobExecution, StepExecution stepExecution) { - String status = (!StringUtil.isBlank(inputParameters.getRemappedFrom())) ? "TRUE": "FALSE"; + String status = (!StringUtil.isBlank(inputParameters.getRemappedFrom())) ? "TRUE" : "FALSE"; return new FlowExecutionStatus(status); } }; } - private Step dummyStep(StepBuilderFactory stepBuilderFactory) { - return stepBuilderFactory.get("step_" + "dummyStep").tasklet(new Tasklet() { - @Override - public RepeatStatus execute(@NonNull StepContribution stepContribution, - @NonNull ChunkContext chunkContext) { - return RepeatStatus.FINISHED; - } - }).build(); + private Step dummyStep(JobRepository jobRepository, PlatformTransactionManager transactionManager) { + return new StepBuilder("step_" + "dummyStep", jobRepository) + .tasklet((stepContribution, chunkContext) -> RepeatStatus.FINISHED, + transactionManager) + .build(); } @Bean(CLUSTERING_FROM_MONGO_JOB) @@ -87,36 +81,35 @@ public Job clusteringFromMongoJob(@Qualifier(CLUSTERING_CLUSTERED_VARIANTS_FROM_ @Qualifier(BACK_PROPAGATE_NEW_RS_STEP) Step backPropagateNewRSStep, // Back-propagate RS in the remapped assembly that were split or merged @Qualifier(BACK_PROPAGATE_SPLIT_OR_MERGED_RS_STEP) - Step backPropagateSplitMergedRSStep, + Step backPropagateSplitMergedRSStep, @Qualifier(JOB_EXECUTION_LISTENER) JobExecutionListener jobExecutionListener, - StepBuilderFactory stepBuilderFactory, - JobBuilderFactory jobBuilderFactory, + JobRepository jobRepository, PlatformTransactionManager transactionManager, InputParameters inputParameters) { JobExecutionDecider jobExecutionDecider = isRemappedAssemblyPresent(inputParameters); - Step dummyStep = dummyStep(stepBuilderFactory); - return jobBuilderFactory.get(CLUSTERING_FROM_MONGO_JOB) + Step dummyStep = dummyStep(jobRepository, transactionManager); + return new JobBuilder(CLUSTERING_FROM_MONGO_JOB, jobRepository) .incrementer(new RunIdIncrementer()) //We need the dummy step here because Spring won't conditionally start the first step .start(dummyStep) .listener(jobExecutionListener) .next(jobExecutionDecider) - .on("TRUE") - .to(new FlowBuilder("remappedAssemblyClusteringFlow") - .start(clusteringClusteredVariantsFromMongoStep) - .next(processRSMergeCandidatesStep) - .next(processRSSplitCandidatesStep) - .next(clearRSMergeAndSplitCandidatesStep) - .next(clusteringNonClusteredVariantsFromMongoStep) - .next(accessioningShutdownStep) - .next(backPropagateNewRSStep) - .next(backPropagateSplitMergedRSStep) - .build()) - .on("*").end() + .on("TRUE") + .to(new FlowBuilder("remappedAssemblyClusteringFlow") + .start(clusteringClusteredVariantsFromMongoStep) + .next(processRSMergeCandidatesStep) + .next(processRSSplitCandidatesStep) + .next(clearRSMergeAndSplitCandidatesStep) + .next(clusteringNonClusteredVariantsFromMongoStep) + .next(accessioningShutdownStep) + .next(backPropagateNewRSStep) + .next(backPropagateSplitMergedRSStep) + .build()) + .on("*").end() .from(jobExecutionDecider) - .on("FALSE") - .to(clusteringNonClusteredVariantsFromMongoStep) - .next(accessioningShutdownStep) - .on("*").end() + .on("FALSE") + .to(clusteringNonClusteredVariantsFromMongoStep) + .next(accessioningShutdownStep) + .on("*").end() .end().build(); } } diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/ProcessRemappedVariantsWithRSJobConfiguration.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/ProcessRemappedVariantsWithRSJobConfiguration.java index 00c0922bb..6a9092b39 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/ProcessRemappedVariantsWithRSJobConfiguration.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/ProcessRemappedVariantsWithRSJobConfiguration.java @@ -18,8 +18,9 @@ import org.springframework.batch.core.Job; import org.springframework.batch.core.Step; import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; -import org.springframework.batch.core.configuration.annotation.JobBuilderFactory; +import org.springframework.batch.core.job.builder.JobBuilder; import org.springframework.batch.core.launch.support.RunIdIncrementer; +import org.springframework.batch.core.repository.JobRepository; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -35,10 +36,10 @@ public class ProcessRemappedVariantsWithRSJobConfiguration { @Bean(PROCESS_REMAPPED_VARIANTS_WITH_RS_JOB) public Job processRemappedVariantsWithRSJob( @Qualifier(CLUSTERING_CLUSTERED_VARIANTS_FROM_MONGO_STEP) Step clusteringClusteredVariantsFromMongoStep, - JobBuilderFactory jobBuilderFactory) { - return jobBuilderFactory.get(PROCESS_REMAPPED_VARIANTS_WITH_RS_JOB) - .incrementer(new RunIdIncrementer()) - .start(clusteringClusteredVariantsFromMongoStep) - .build(); + JobRepository jobRepository) { + return new JobBuilder(PROCESS_REMAPPED_VARIANTS_WITH_RS_JOB, jobRepository) + .incrementer(new RunIdIncrementer()) + .start(clusteringClusteredVariantsFromMongoStep) + .build(); } } diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/RSAccessionRecoveryJobConfiguration.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/RSAccessionRecoveryJobConfiguration.java index d9d80cafc..7070c9dbc 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/RSAccessionRecoveryJobConfiguration.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/RSAccessionRecoveryJobConfiguration.java @@ -4,8 +4,9 @@ import org.springframework.batch.core.JobExecutionListener; import org.springframework.batch.core.Step; import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; -import org.springframework.batch.core.configuration.annotation.JobBuilderFactory; +import org.springframework.batch.core.job.builder.JobBuilder; import org.springframework.batch.core.launch.support.RunIdIncrementer; +import org.springframework.batch.core.repository.JobRepository; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; @@ -28,8 +29,8 @@ public class RSAccessionRecoveryJobConfiguration { private JobExecutionListener monotonicAccessionRecoveryAgentCategoryRSJobListener; @Bean(RS_ACCESSION_RECOVERY_JOB) - public Job createMonotonicAccessionRecoveryAgentCategoryRSJob(JobBuilderFactory jobBuilderFactory) { - return jobBuilderFactory.get(RS_ACCESSION_RECOVERY_JOB) + public Job createMonotonicAccessionRecoveryAgentCategoryRSJob(JobRepository jobRepository) { + return new JobBuilder(RS_ACCESSION_RECOVERY_JOB, jobRepository) .incrementer(new RunIdIncrementer()) .start(monotonicAccessionRecoveryAgentCategoryRSStep) .listener(monotonicAccessionRecoveryAgentCategoryRSJobListener) diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/StudyClusteringJobConfiguration.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/StudyClusteringJobConfiguration.java index 152e04ceb..36344d7e6 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/StudyClusteringJobConfiguration.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/StudyClusteringJobConfiguration.java @@ -19,8 +19,9 @@ import org.springframework.batch.core.JobExecutionListener; import org.springframework.batch.core.Step; import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; -import org.springframework.batch.core.configuration.annotation.JobBuilderFactory; +import org.springframework.batch.core.job.builder.JobBuilder; import org.springframework.batch.core.launch.support.RunIdIncrementer; +import org.springframework.batch.core.repository.JobRepository; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -38,12 +39,12 @@ public class StudyClusteringJobConfiguration { public Job studyClusteringJob(@Qualifier(STUDY_CLUSTERING_STEP) Step clusteringStep, @Qualifier(ACCESSIONING_SHUTDOWN_STEP) Step accessioningShutdownStep, @Qualifier(JOB_EXECUTION_LISTENER) JobExecutionListener jobExecutionListener, - JobBuilderFactory jobBuilderFactory) { - return jobBuilderFactory.get(STUDY_CLUSTERING_JOB) - .incrementer(new RunIdIncrementer()) - .start(clusteringStep) - .next(accessioningShutdownStep) - .listener(jobExecutionListener) - .build(); + JobRepository jobRepository) { + return new JobBuilder(STUDY_CLUSTERING_JOB, jobRepository) + .incrementer(new RunIdIncrementer()) + .start(clusteringStep) + .next(accessioningShutdownStep) + .listener(jobExecutionListener) + .build(); } } diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/qc/ClusteringQCJobConfiguration.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/qc/ClusteringQCJobConfiguration.java index 4308574bc..69717ddfd 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/qc/ClusteringQCJobConfiguration.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/qc/ClusteringQCJobConfiguration.java @@ -19,9 +19,10 @@ import org.springframework.batch.core.Step; import org.springframework.batch.core.StepExecutionListener; import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; -import org.springframework.batch.core.configuration.annotation.JobBuilderFactory; -import org.springframework.batch.core.configuration.annotation.StepBuilderFactory; +import org.springframework.batch.core.job.builder.JobBuilder; import org.springframework.batch.core.launch.support.RunIdIncrementer; +import org.springframework.batch.core.repository.JobRepository; +import org.springframework.batch.core.step.builder.StepBuilder; import org.springframework.batch.item.ItemProcessor; import org.springframework.batch.item.ItemStreamReader; import org.springframework.batch.item.ItemWriter; @@ -30,6 +31,7 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.transaction.PlatformTransactionManager; import uk.ac.ebi.eva.accession.clustering.batch.io.ClusteringWriter; import uk.ac.ebi.eva.accession.clustering.batch.io.qc.ExtraneousRSReporter; import uk.ac.ebi.eva.accession.clustering.batch.io.qc.PendingMergeSplitReporter; @@ -91,14 +93,14 @@ public ExtraneousRSReporter extraneousRSReporter(InputParameters parameters, Mon public Step reportUnclusteredSSAndPendingMergeSplitStep( @Qualifier(REMAPPED_SS_READER) ItemStreamReader remappedSSReader, @Qualifier(REPORT_UNCLUSTERED_SS_PROCESSOR) - ItemProcessor reportUnclusteredSSProcessor, + ItemProcessor reportUnclusteredSSProcessor, @Qualifier(PENDING_MERGE_AND_SPLIT_REPORTER) - ItemWriter pendingMergeSplitReporter, + ItemWriter pendingMergeSplitReporter, @Qualifier(PROGRESS_LISTENER) StepExecutionListener progressListener, - StepBuilderFactory stepBuilderFactory, + JobRepository jobRepository, PlatformTransactionManager transactionManager, SimpleCompletionPolicy chunkSizeCompletionPolicy) { - return stepBuilderFactory.get(REPORT_UNCLUSTERED_SS_AND_PENDING_MERGES_AND_SPLITS_STEP) - .chunk(chunkSizeCompletionPolicy) + return new StepBuilder(REPORT_UNCLUSTERED_SS_AND_PENDING_MERGES_AND_SPLITS_STEP, jobRepository) + .chunk(chunkSizeCompletionPolicy, transactionManager) .reader(remappedSSReader) .processor(reportUnclusteredSSProcessor) .writer(pendingMergeSplitReporter) @@ -111,12 +113,12 @@ public Step reportUnclusteredSSAndPendingMergeSplitStep( public Step reportExtraneousRSStep( @Qualifier(REMAPPED_RS_READER) ItemStreamReader remappedRSReader, @Qualifier(EXTRANEOUS_RS_REPORTER) - ItemWriter extraneousRSReporter, + ItemWriter extraneousRSReporter, @Qualifier(PROGRESS_LISTENER) StepExecutionListener progressListener, - StepBuilderFactory stepBuilderFactory, + JobRepository jobRepository, PlatformTransactionManager transactionManager, SimpleCompletionPolicy chunkSizeCompletionPolicy) { - return stepBuilderFactory.get(REPORT_EXTRANEOUS_RS_STEP) - .chunk(chunkSizeCompletionPolicy) + return new StepBuilder(REPORT_EXTRANEOUS_RS_STEP, jobRepository) + .chunk(chunkSizeCompletionPolicy, transactionManager) .reader(remappedRSReader) .writer(extraneousRSReporter) .listener(progressListener) @@ -126,11 +128,11 @@ public Step reportExtraneousRSStep( @Bean(CLUSTERING_QC_JOB) public Job ClusteringQCJob( @Qualifier(REPORT_UNCLUSTERED_SS_AND_PENDING_MERGES_AND_SPLITS_STEP) - Step reportUnclusteredSSAndPendingMergeSplitStep, + Step reportUnclusteredSSAndPendingMergeSplitStep, @Qualifier(REPORT_EXTRANEOUS_RS_STEP) - Step reportExtraneousRSStep, - JobBuilderFactory jobBuilderFactory) { - return jobBuilderFactory.get(CLUSTERING_QC_JOB) + Step reportExtraneousRSStep, + JobRepository jobRepository) { + return new JobBuilder(CLUSTERING_QC_JOB, jobRepository) .incrementer(new RunIdIncrementer()) .start(reportUnclusteredSSAndPendingMergeSplitStep) .next(reportExtraneousRSStep) diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/qc/DuplicateRSAccQCJobConfiguration.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/qc/DuplicateRSAccQCJobConfiguration.java index 1323233dd..48294ee59 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/qc/DuplicateRSAccQCJobConfiguration.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/qc/DuplicateRSAccQCJobConfiguration.java @@ -3,8 +3,9 @@ import org.springframework.batch.core.Job; import org.springframework.batch.core.Step; import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; -import org.springframework.batch.core.configuration.annotation.JobBuilderFactory; +import org.springframework.batch.core.job.builder.JobBuilder; import org.springframework.batch.core.launch.support.RunIdIncrementer; +import org.springframework.batch.core.repository.JobRepository; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; @@ -22,8 +23,8 @@ public class DuplicateRSAccQCJobConfiguration { private Step duplicateRSAccQCStep; @Bean(DUPLICATE_RS_ACC_QC_JOB) - public Job duplicateRSAccQCJob(JobBuilderFactory jobBuilderFactory) { - return jobBuilderFactory.get(DUPLICATE_RS_ACC_QC_JOB) + public Job duplicateRSAccQCJob(JobRepository jobRepository) { + return new JobBuilder(DUPLICATE_RS_ACC_QC_JOB, jobRepository) .incrementer(new RunIdIncrementer()) .start(duplicateRSAccQCStep) .build(); diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/qc/NewClusteredVariantsQCJobConfiguration.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/qc/NewClusteredVariantsQCJobConfiguration.java index 9c3cb7d3e..beddc0eaa 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/qc/NewClusteredVariantsQCJobConfiguration.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/qc/NewClusteredVariantsQCJobConfiguration.java @@ -18,9 +18,10 @@ import org.springframework.batch.core.Job; import org.springframework.batch.core.Step; import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; -import org.springframework.batch.core.configuration.annotation.JobBuilderFactory; -import org.springframework.batch.core.configuration.annotation.StepBuilderFactory; +import org.springframework.batch.core.job.builder.JobBuilder; import org.springframework.batch.core.launch.support.RunIdIncrementer; +import org.springframework.batch.core.repository.JobRepository; +import org.springframework.batch.core.step.builder.StepBuilder; import org.springframework.batch.item.ItemStreamReader; import org.springframework.batch.item.ItemWriter; import org.springframework.batch.item.file.FlatFileItemReader; @@ -31,7 +32,7 @@ import org.springframework.context.annotation.Configuration; import org.springframework.core.io.FileSystemResource; import org.springframework.data.mongodb.core.MongoTemplate; - +import org.springframework.transaction.PlatformTransactionManager; import uk.ac.ebi.eva.accession.clustering.batch.io.qc.MissingCveReporter; import uk.ac.ebi.eva.accession.clustering.batch.io.qc.RSHashPair; @@ -79,20 +80,19 @@ public MissingCveReporter missingCveReporter(MongoTemplate mongoTemplate) { public Step reportMissingCveStep( @Qualifier(RS_REPORT_READER) ItemStreamReader rsReportReader, @Qualifier(MISSING_CVE_REPORTER) ItemWriter missingCveReporter, - StepBuilderFactory stepBuilderFactory, + JobRepository jobRepository, PlatformTransactionManager transactionManager, SimpleCompletionPolicy chunkSizeCompletionPolicy) { - return stepBuilderFactory.get(REPORT_MISSING_CVE_STEP) - .chunk(chunkSizeCompletionPolicy) - .reader(rsReportReader) - .writer(missingCveReporter) - .build(); + return new StepBuilder(REPORT_MISSING_CVE_STEP, jobRepository) + .chunk(chunkSizeCompletionPolicy, transactionManager) + .reader(rsReportReader) + .writer(missingCveReporter) + .build(); } @Bean(NEW_CLUSTERED_VARIANTS_QC_JOB) - public Job ClusteringQCJob( - @Qualifier(REPORT_MISSING_CVE_STEP) Step reportMissingCveStep, - JobBuilderFactory jobBuilderFactory) { - return jobBuilderFactory.get(NEW_CLUSTERED_VARIANTS_QC_JOB) + public Job ClusteringQCJob(@Qualifier(REPORT_MISSING_CVE_STEP) Step reportMissingCveStep, + JobRepository jobRepository) { + return new JobBuilder(NEW_CLUSTERED_VARIANTS_QC_JOB, jobRepository) .incrementer(new RunIdIncrementer()) .start(reportMissingCveStep) .build(); diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/steps/AccessioningShutdownStepConfiguration.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/steps/AccessioningShutdownStepConfiguration.java index 1c67b1b0e..766cf292b 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/steps/AccessioningShutdownStepConfiguration.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/steps/AccessioningShutdownStepConfiguration.java @@ -2,10 +2,12 @@ import org.springframework.batch.core.Step; import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; -import org.springframework.batch.core.configuration.annotation.StepBuilderFactory; +import org.springframework.batch.core.repository.JobRepository; +import org.springframework.batch.core.step.builder.StepBuilder; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.transaction.PlatformTransactionManager; import uk.ac.ebi.eva.accession.core.service.nonhuman.ClusteredVariantAccessioningService; import uk.ac.ebi.eva.accession.core.service.nonhuman.SubmittedVariantAccessioningService; @@ -21,13 +23,13 @@ public class AccessioningShutdownStepConfiguration { private ClusteredVariantAccessioningService clusteredVariantAccessioningService; @Bean(ACCESSIONING_SHUTDOWN_STEP) - public Step accessioningShutDownStep(StepBuilderFactory stepBuilderFactory) { - return stepBuilderFactory.get(ACCESSIONING_SHUTDOWN_STEP) + public Step accessioningShutDownStep(JobRepository jobRepository, PlatformTransactionManager transactionManager) { + return new StepBuilder(ACCESSIONING_SHUTDOWN_STEP, jobRepository) .tasklet((contribution, chunkContext) -> { submittedVariantAccessioningService.shutDownAccessionGenerator(); clusteredVariantAccessioningService.shutDownAccessionGenerator(); return null; - }) + }, transactionManager) .build(); } } \ No newline at end of file diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/steps/ClusteringFromMongoStepConfiguration.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/steps/ClusteringFromMongoStepConfiguration.java index 3e1701265..41417be99 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/steps/ClusteringFromMongoStepConfiguration.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/steps/ClusteringFromMongoStepConfiguration.java @@ -19,7 +19,8 @@ import org.springframework.batch.core.Step; import org.springframework.batch.core.StepExecutionListener; import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; -import org.springframework.batch.core.configuration.annotation.StepBuilderFactory; +import org.springframework.batch.core.repository.JobRepository; +import org.springframework.batch.core.step.builder.StepBuilder; import org.springframework.batch.core.step.tasklet.TaskletStep; import org.springframework.batch.item.ItemReader; import org.springframework.batch.item.ItemStreamReader; @@ -28,38 +29,38 @@ import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; - +import org.springframework.transaction.PlatformTransactionManager; import uk.ac.ebi.eva.accession.clustering.batch.io.ListOfListItemWriter; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantOperationEntity; import java.util.List; -import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.CLUSTERED_CLUSTERING_WRITER_JOB_EXECUTION_SETTER; -import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.NON_CLUSTERED_CLUSTERING_WRITER_JOB_EXECUTION_SETTER; -import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.RS_SPLIT_WRITER_JOB_EXECUTION_SETTER; -import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.STUDY_CLUSTERING_MONGO_READER; -import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.STUDY_CLUSTERING_STEP; -import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.TARGET_SS_READER_FOR_NEW_BACKPROP_RS; -import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.TARGET_SS_READER_FOR_SPLIT_OR_MERGED_BACKPROP_RS; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.BACK_PROPAGATED_RS_WRITER; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.BACK_PROPAGATE_NEW_RS_STEP; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.BACK_PROPAGATE_SPLIT_OR_MERGED_RS_STEP; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.CLEAR_RS_MERGE_AND_SPLIT_CANDIDATES; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.CLEAR_RS_MERGE_AND_SPLIT_CANDIDATES_STEP; +import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.CLUSTERED_CLUSTERING_WRITER; +import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.CLUSTERED_CLUSTERING_WRITER_JOB_EXECUTION_SETTER; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.CLUSTERED_VARIANTS_MONGO_READER; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.CLUSTERING_CLUSTERED_VARIANTS_FROM_MONGO_STEP; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.CLUSTERING_NON_CLUSTERED_VARIANTS_FROM_MONGO_STEP; -import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.CLUSTERED_CLUSTERING_WRITER; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.NON_CLUSTERED_CLUSTERING_WRITER; +import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.NON_CLUSTERED_CLUSTERING_WRITER_JOB_EXECUTION_SETTER; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.NON_CLUSTERED_VARIANTS_MONGO_READER; -import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.PROCESS_RS_SPLIT_CANDIDATES_STEP; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.PROCESS_RS_MERGE_CANDIDATES_STEP; +import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.PROCESS_RS_SPLIT_CANDIDATES_STEP; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.PROGRESS_LISTENER; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.RS_MERGE_CANDIDATES_READER; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.RS_MERGE_WRITER; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.RS_SPLIT_CANDIDATES_READER; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.RS_SPLIT_WRITER; +import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.RS_SPLIT_WRITER_JOB_EXECUTION_SETTER; +import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.STUDY_CLUSTERING_MONGO_READER; +import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.STUDY_CLUSTERING_STEP; +import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.TARGET_SS_READER_FOR_NEW_BACKPROP_RS; +import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.TARGET_SS_READER_FOR_SPLIT_OR_MERGED_BACKPROP_RS; @Configuration @EnableBatchProcessing @@ -71,10 +72,10 @@ public Step clusteringClusteredVariantStepMongoReader( @Qualifier(CLUSTERED_CLUSTERING_WRITER) ItemWriter submittedVariantWriter, @Qualifier(PROGRESS_LISTENER) StepExecutionListener progressListener, @Qualifier(CLUSTERED_CLUSTERING_WRITER_JOB_EXECUTION_SETTER) StepExecutionListener clusteredClusteringWriterJobExecutionSetter, - StepBuilderFactory stepBuilderFactory, + JobRepository jobRepository, PlatformTransactionManager transactionManager, SimpleCompletionPolicy chunkSizeCompletionPolicy) { - TaskletStep step = stepBuilderFactory.get(CLUSTERING_CLUSTERED_VARIANTS_FROM_MONGO_STEP) - .chunk(chunkSizeCompletionPolicy) + TaskletStep step = new StepBuilder(CLUSTERING_CLUSTERED_VARIANTS_FROM_MONGO_STEP, jobRepository) + .chunk(chunkSizeCompletionPolicy, transactionManager) .reader(mongoReader) .writer(submittedVariantWriter) .listener(progressListener) @@ -86,51 +87,49 @@ public Step clusteringClusteredVariantStepMongoReader( @Bean(PROCESS_RS_MERGE_CANDIDATES_STEP) public Step processRSMergeCandidatesStep( @Qualifier(RS_MERGE_CANDIDATES_READER) - ItemReader rsMergeCandidatesReader, + ItemReader rsMergeCandidatesReader, @Qualifier(RS_MERGE_WRITER) ItemWriter rsMergeWriter, @Qualifier(PROGRESS_LISTENER) StepExecutionListener progressListener, - StepBuilderFactory stepBuilderFactory, + JobRepository jobRepository, PlatformTransactionManager transactionManager, SimpleCompletionPolicy chunkSizeCompletionPolicy) { - TaskletStep step = stepBuilderFactory.get(PROCESS_RS_MERGE_CANDIDATES_STEP) - .chunk( - chunkSizeCompletionPolicy) - .reader(rsMergeCandidatesReader) - .writer(rsMergeWriter) - .listener(progressListener) - .build(); + TaskletStep step = new StepBuilder(PROCESS_RS_MERGE_CANDIDATES_STEP, jobRepository) + .chunk(chunkSizeCompletionPolicy, transactionManager) + .reader(rsMergeCandidatesReader) + .writer(rsMergeWriter) + .listener(progressListener) + .build(); return step; } @Bean(PROCESS_RS_SPLIT_CANDIDATES_STEP) public Step processRSSplitCandidatesStep( @Qualifier(RS_SPLIT_CANDIDATES_READER) - ItemReader rsSplitCandidatesReader, + ItemReader rsSplitCandidatesReader, @Qualifier(RS_SPLIT_WRITER) ItemWriter rsSplitWriter, @Qualifier(PROGRESS_LISTENER) StepExecutionListener progressListener, @Qualifier(RS_SPLIT_WRITER_JOB_EXECUTION_SETTER) StepExecutionListener rsSplitWriterJobExecutionSetter, - StepBuilderFactory stepBuilderFactory, + JobRepository jobRepository, PlatformTransactionManager transactionManager, SimpleCompletionPolicy chunkSizeCompletionPolicy) { - TaskletStep step = stepBuilderFactory.get(PROCESS_RS_SPLIT_CANDIDATES_STEP) - .chunk( - chunkSizeCompletionPolicy) - .reader(rsSplitCandidatesReader) - .writer(rsSplitWriter) - .listener(progressListener) - .listener(rsSplitWriterJobExecutionSetter) - .build(); + TaskletStep step = new StepBuilder(PROCESS_RS_SPLIT_CANDIDATES_STEP, jobRepository) + .chunk(chunkSizeCompletionPolicy, transactionManager) + .reader(rsSplitCandidatesReader) + .writer(rsSplitWriter) + .listener(progressListener) + .listener(rsSplitWriterJobExecutionSetter) + .build(); return step; } @Bean(CLEAR_RS_MERGE_AND_SPLIT_CANDIDATES_STEP) public Step clearRSMergeAndSplitCandidatesStep( @Qualifier(CLEAR_RS_MERGE_AND_SPLIT_CANDIDATES) ItemWriter clearRSMergeAndSplitCandidates, - StepBuilderFactory stepBuilderFactory, + JobRepository jobRepository, PlatformTransactionManager transactionManager, SimpleCompletionPolicy chunkSizeCompletionPolicy) { - TaskletStep step = stepBuilderFactory.get(CLEAR_RS_MERGE_AND_SPLIT_CANDIDATES_STEP) - .chunk(chunkSizeCompletionPolicy) - .reader(new SingleItemReader()) - .writer(clearRSMergeAndSplitCandidates) - .build(); + TaskletStep step = new StepBuilder(CLEAR_RS_MERGE_AND_SPLIT_CANDIDATES_STEP, jobRepository) + .chunk(chunkSizeCompletionPolicy, transactionManager) + .reader(new SingleItemReader()) + .writer(clearRSMergeAndSplitCandidates) + .build(); return step; } @@ -139,6 +138,7 @@ public Step clearRSMergeAndSplitCandidatesStep( // so that the writer to clear RS merge and split candidate entries can proceed public static class SingleItemReader implements ItemReader { static boolean firstTime = true; + @Override public Object read() throws Exception { if (firstTime) { @@ -155,10 +155,10 @@ public Step clusteringNonClusteredVariantStepMongoReader( @Qualifier(NON_CLUSTERED_CLUSTERING_WRITER) ItemWriter submittedVariantWriter, @Qualifier(PROGRESS_LISTENER) StepExecutionListener progressListener, @Qualifier(NON_CLUSTERED_CLUSTERING_WRITER_JOB_EXECUTION_SETTER) StepExecutionListener nonClusteredClusteringWriterJobExecutionSetter, - StepBuilderFactory stepBuilderFactory, + JobRepository jobRepository, PlatformTransactionManager transactionManager, SimpleCompletionPolicy chunkSizeCompletionPolicy) { - TaskletStep step = stepBuilderFactory.get(CLUSTERING_NON_CLUSTERED_VARIANTS_FROM_MONGO_STEP) - .chunk(chunkSizeCompletionPolicy) + TaskletStep step = new StepBuilder(CLUSTERING_NON_CLUSTERED_VARIANTS_FROM_MONGO_STEP, jobRepository) + .chunk(chunkSizeCompletionPolicy, transactionManager) .reader(mongoReader) .writer(submittedVariantWriter) .listener(progressListener) @@ -170,34 +170,32 @@ public Step clusteringNonClusteredVariantStepMongoReader( @Bean(BACK_PROPAGATE_NEW_RS_STEP) public Step backPropagateNewRSStep( @Qualifier(TARGET_SS_READER_FOR_NEW_BACKPROP_RS) - ItemStreamReader backPropagatedNewRSTargetReader, + ItemStreamReader backPropagatedNewRSTargetReader, @Qualifier(BACK_PROPAGATED_RS_WRITER) ItemWriter backPropagatedRSWriter, @Qualifier(PROGRESS_LISTENER) StepExecutionListener progressListener, - StepBuilderFactory stepBuilderFactory, + JobRepository jobRepository, PlatformTransactionManager transactionManager, SimpleCompletionPolicy chunkSizeCompletionPolicy) { - TaskletStep step = stepBuilderFactory.get(BACK_PROPAGATE_NEW_RS_STEP) - .chunk( - chunkSizeCompletionPolicy) - .reader(backPropagatedNewRSTargetReader) - .writer(backPropagatedRSWriter) - .listener(progressListener) - .build(); + TaskletStep step = new StepBuilder(BACK_PROPAGATE_NEW_RS_STEP, jobRepository) + .chunk(chunkSizeCompletionPolicy, transactionManager) + .reader(backPropagatedNewRSTargetReader) + .writer(backPropagatedRSWriter) + .listener(progressListener) + .build(); return step; } @Bean(BACK_PROPAGATE_SPLIT_OR_MERGED_RS_STEP) public Step backPropagateSplitAndMergedRSStep( @Qualifier(TARGET_SS_READER_FOR_SPLIT_OR_MERGED_BACKPROP_RS) - ItemStreamReader> splitOrMergedRSReader, + ItemStreamReader> splitOrMergedRSReader, @Qualifier(BACK_PROPAGATED_RS_WRITER) ItemWriter backPropagatedRSWriter, @Qualifier(PROGRESS_LISTENER) StepExecutionListener progressListener, - StepBuilderFactory stepBuilderFactory, + JobRepository jobRepository, PlatformTransactionManager transactionManager, SimpleCompletionPolicy chunkSizeCompletionPolicy) { - TaskletStep step = stepBuilderFactory.get(BACK_PROPAGATE_SPLIT_OR_MERGED_RS_STEP) - ., List>chunk( - chunkSizeCompletionPolicy) + TaskletStep step = new StepBuilder(BACK_PROPAGATE_SPLIT_OR_MERGED_RS_STEP, jobRepository) + ., List>chunk(chunkSizeCompletionPolicy, transactionManager) .reader(splitOrMergedRSReader) - // Spring needs this wrapping to flatten the List> from the processor + // Spring needs this wrapping to flatten the List> from the processor .writer(new ListOfListItemWriter<>(backPropagatedRSWriter)) .listener(progressListener) .build(); @@ -210,15 +208,15 @@ public Step studyClusteringStep( @Qualifier(NON_CLUSTERED_CLUSTERING_WRITER) ItemWriter submittedVariantWriter, @Qualifier(PROGRESS_LISTENER) StepExecutionListener progressListener, @Qualifier(NON_CLUSTERED_CLUSTERING_WRITER_JOB_EXECUTION_SETTER) StepExecutionListener nonClusteredClusteringWriterJobExecutionSetter, - StepBuilderFactory stepBuilderFactory, + JobRepository jobRepository, PlatformTransactionManager transactionManager, SimpleCompletionPolicy chunkSizeCompletionPolicy) { - TaskletStep step = stepBuilderFactory.get(STUDY_CLUSTERING_STEP) - .chunk(chunkSizeCompletionPolicy) - .reader(mongoReader) - .writer(submittedVariantWriter) - .listener(progressListener) - .listener(nonClusteredClusteringWriterJobExecutionSetter) - .build(); + TaskletStep step = new StepBuilder(STUDY_CLUSTERING_STEP, jobRepository) + .chunk(chunkSizeCompletionPolicy, transactionManager) + .reader(mongoReader) + .writer(submittedVariantWriter) + .listener(progressListener) + .listener(nonClusteredClusteringWriterJobExecutionSetter) + .build(); return step; } } diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/steps/RSAccessionRecoveryStepConfiguration.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/steps/RSAccessionRecoveryStepConfiguration.java index a0514439d..f484eda83 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/steps/RSAccessionRecoveryStepConfiguration.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/steps/RSAccessionRecoveryStepConfiguration.java @@ -2,11 +2,13 @@ import org.springframework.batch.core.Step; import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; -import org.springframework.batch.core.configuration.annotation.StepBuilderFactory; +import org.springframework.batch.core.repository.JobRepository; +import org.springframework.batch.core.step.builder.StepBuilder; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.transaction.PlatformTransactionManager; import uk.ac.ebi.eva.accession.clustering.batch.recovery.RSAccessionRecoveryService; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.RS_ACCESSION_RECOVERY_SERVICE; @@ -20,12 +22,13 @@ public class RSAccessionRecoveryStepConfiguration { private RSAccessionRecoveryService RSAccessionRecoveryService; @Bean(RS_ACCESSION_RECOVERY_STEP) - public Step monotonicAccessionRecoveryAgentCategoryRSStep(StepBuilderFactory stepBuilderFactory) { - return stepBuilderFactory.get(RS_ACCESSION_RECOVERY_STEP) + public Step monotonicAccessionRecoveryAgentCategoryRSStep(JobRepository jobRepository, + PlatformTransactionManager transactionManager) { + return new StepBuilder(RS_ACCESSION_RECOVERY_STEP, jobRepository) .tasklet((contribution, chunkContext) -> { RSAccessionRecoveryService.runRecoveryForCategoryRS(); return null; - }) + }, transactionManager) .build(); } } diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/steps/qc/DuplicateRSAccQCStepConfiguration.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/steps/qc/DuplicateRSAccQCStepConfiguration.java index 9af162201..2c0afce12 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/steps/qc/DuplicateRSAccQCStepConfiguration.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/steps/qc/DuplicateRSAccQCStepConfiguration.java @@ -2,7 +2,8 @@ import org.springframework.batch.core.Step; import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; -import org.springframework.batch.core.configuration.annotation.StepBuilderFactory; +import org.springframework.batch.core.repository.JobRepository; +import org.springframework.batch.core.step.builder.StepBuilder; import org.springframework.batch.core.step.tasklet.TaskletStep; import org.springframework.batch.item.ItemProcessor; import org.springframework.batch.item.ItemStreamReader; @@ -11,6 +12,7 @@ import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.transaction.PlatformTransactionManager; import uk.ac.ebi.eva.accession.clustering.batch.io.qc.DuplicateRSAccQCResult; import java.util.List; @@ -37,11 +39,11 @@ public class DuplicateRSAccQCStepConfiguration { private ItemWriter> duplicateRSAccQCWriter; @Bean(DUPLICATE_RS_ACC_QC_STEP) - public Step duplicateRSAccQCStep(StepBuilderFactory stepBuilderFactory) { - TaskletStep step = stepBuilderFactory.get(DUPLICATE_RS_ACC_QC_STEP) + public Step duplicateRSAccQCStep(JobRepository jobRepository, PlatformTransactionManager transactionManager) { + TaskletStep step = new StepBuilder(DUPLICATE_RS_ACC_QC_STEP, jobRepository) // hardcoded the chunk size as 1, as the reader takes care of accumulating // and sending the chunk size (defined in properties file) elements to the processor - ., List>chunk(1) + ., List>chunk(1, transactionManager) .reader(duplicateRSAccFileReader) .processor(duplicateRSAccQCProcessor) .writer(duplicateRSAccQCWriter) diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/metric/ClusteringMetricCompute.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/metric/ClusteringMetricCompute.java index 1491f700f..d32187071 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/metric/ClusteringMetricCompute.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/metric/ClusteringMetricCompute.java @@ -1,7 +1,8 @@ package uk.ac.ebi.eva.accession.clustering.metric; -import org.codehaus.jettison.json.JSONException; -import org.codehaus.jettison.json.JSONObject; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; import org.springframework.web.client.RestTemplate; import uk.ac.ebi.eva.metrics.count.CountServiceParameters; import uk.ac.ebi.eva.metrics.metric.BaseMetricCompute; @@ -11,6 +12,8 @@ import java.util.stream.Collectors; public class ClusteringMetricCompute extends BaseMetricCompute { + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + private static final String PROCESS = "clustering"; private final String assembly; @@ -33,12 +36,12 @@ public List getMetrics() { public String getIdentifier() { try { - JSONObject identifier = new JSONObject(); + ObjectNode identifier = OBJECT_MAPPER.createObjectNode(); identifier.put("assembly", assembly); - identifier.put("projects", projects); - return identifier.toString(); - } catch (JSONException jsonException) { - throw new RuntimeException("Could not create Identifier for Clustering Counts. Error ", jsonException); + identifier.putPOJO("projects", projects); + return OBJECT_MAPPER.writeValueAsString(identifier); + } catch (JsonProcessingException e) { + throw new RuntimeException("Could not create Identifier for Clustering Counts. Error ", e); } } diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/parameters/InputParameters.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/parameters/InputParameters.java index a2358de0b..c2128afb8 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/parameters/InputParameters.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/parameters/InputParameters.java @@ -15,7 +15,6 @@ */ package uk.ac.ebi.eva.accession.clustering.parameters; -import org.apache.commons.collections.CollectionUtils; import org.springframework.batch.core.JobParameters; import org.springframework.batch.core.JobParametersBuilder; import org.springframework.batch.core.JobParametersInvalidException; @@ -95,7 +94,7 @@ public JobParameters toJobParameters() throws JobParametersInvalidException { if (projects.stream().anyMatch(s -> s.contains(","))) { throw new JobParametersInvalidException("Can't have commas in project accessions"); } - String projectsString = CollectionUtils.isEmpty(projects) ? "" : String.join(",", projects); + String projectsString = (projects == null || projects.isEmpty()) ? "" : String.join(",", projects); if (projectsString.length() > 250) { throw new JobParametersInvalidException("Max length of projects parameter is 250 characters"); } diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/runner/ClusteringCommandLineRunner.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/runner/ClusteringCommandLineRunner.java index 8b089f129..25e60ba7f 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/runner/ClusteringCommandLineRunner.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/runner/ClusteringCommandLineRunner.java @@ -25,7 +25,6 @@ import org.springframework.batch.core.JobParametersInvalidException; import org.springframework.batch.core.explore.JobExplorer; import org.springframework.batch.core.launch.JobLauncher; -import org.springframework.batch.core.launch.JobParametersNotFoundException; import org.springframework.batch.core.repository.JobExecutionAlreadyRunningException; import org.springframework.batch.core.repository.JobInstanceAlreadyCompleteException; import org.springframework.batch.core.repository.JobRepository; @@ -33,7 +32,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.ExitCodeGenerator; -import org.springframework.boot.autoconfigure.batch.JobLauncherCommandLineRunner; +import org.springframework.boot.autoconfigure.batch.JobLauncherApplicationRunner; import org.springframework.context.ApplicationEventPublisherAware; import org.springframework.stereotype.Component; import org.springframework.util.PatternMatchUtils; @@ -49,7 +48,7 @@ import java.util.Collection; @Component -public class ClusteringCommandLineRunner extends JobLauncherCommandLineRunner implements +public class ClusteringCommandLineRunner extends JobLauncherApplicationRunner implements ApplicationEventPublisherAware, ExitCodeGenerator { private static final Logger logger = LoggerFactory.getLogger(ClusteringCommandLineRunner.class); @@ -90,9 +89,9 @@ public void setJobs(Collection jobs) { } @Override - public void setJobNames(String jobName) { + public void setJobName(String jobName) { this.jobName = jobName; - super.setJobNames(jobName); + super.setJobName(jobName); } @Override @@ -116,16 +115,15 @@ public void run(String... args) throws JobExecutionException { JobStatusManager.checkIfPropertiesHaveBeenProvided(jobParameters); if (inputParameters.isForceRestart()) { JobExecution previousJobExecution = CommandLineRunnerUtils.getLastJobExecution(jobName, jobExplorer, - jobParameters); + jobParameters); CommandLineRunnerUtils.markPreviousJobAsFailed(jobName, jobRepository, previousJobExecution != null ? previousJobExecution.getJobParameters() : jobParameters); - } - else { + } else { jobParameters = CommandLineRunnerUtils.addRunIDToJobParameters(jobName, jobExplorer, jobParameters); } launchJob(jobParameters); } catch (NoJobToExecuteException | NoParametersHaveBeenPassedException | NoPreviousJobExecutionException - | UnknownJobException | JobParametersInvalidException | JobExecutionAlreadyRunningException e) { + | UnknownJobException | JobParametersInvalidException | JobExecutionAlreadyRunningException e) { logger.error(e.getMessage()); logger.debug("Error trace", e); abnormalExit = true; @@ -147,8 +145,7 @@ private void launchJob(JobParameters jobParameters) throws JobExecutionException @Override protected void execute(Job job, JobParameters jobParameters) throws JobExecutionAlreadyRunningException, - JobRestartException, JobInstanceAlreadyCompleteException, JobParametersInvalidException, - JobParametersNotFoundException { + JobRestartException, JobInstanceAlreadyCompleteException, JobParametersInvalidException { logger.info("Running job '" + jobName + "' with parameters: " + jobParameters); super.execute(job, jobParameters); } diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/BackPropagatedRSWriterTest.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/BackPropagatedRSWriterTest.java index f4df28eaf..a255bf923 100644 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/BackPropagatedRSWriterTest.java +++ b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/BackPropagatedRSWriterTest.java @@ -15,49 +15,53 @@ */ package uk.ac.ebi.eva.accession.clustering.batch.io; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; -import org.junit.*; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.batch.item.Chunk; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.context.ApplicationContext; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.ampt2d.commons.accession.core.exceptions.AccessionCouldNotBeGeneratedException; import uk.ac.ebi.ampt2d.commons.accession.core.models.AccessionWrapper; import uk.ac.ebi.ampt2d.commons.accession.hashing.SHA1HashingFunction; import uk.ac.ebi.eva.accession.clustering.test.configuration.BatchTestConfiguration; -import uk.ac.ebi.eva.accession.clustering.test.rule.FixSpringMongoDbRule; +import uk.ac.ebi.eva.accession.clustering.test.configuration.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.model.ClusteredVariant; import uk.ac.ebi.eva.accession.core.model.IClusteredVariant; import uk.ac.ebi.eva.accession.core.model.ISubmittedVariant; import uk.ac.ebi.eva.accession.core.model.SubmittedVariant; -import uk.ac.ebi.eva.accession.core.model.dbsnp.*; -import uk.ac.ebi.eva.accession.core.model.eva.*; +import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpClusteredVariantEntity; +import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpSubmittedVariantEntity; +import uk.ac.ebi.eva.accession.core.model.eva.ClusteredVariantEntity; +import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; import uk.ac.ebi.eva.accession.core.service.nonhuman.SubmittedVariantAccessioningService; import uk.ac.ebi.eva.accession.core.summary.ClusteredVariantSummaryFunction; import uk.ac.ebi.eva.accession.core.summary.SubmittedVariantSummaryFunction; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.commons.core.models.VariantType; import uk.ac.ebi.eva.metrics.metric.MetricCompute; -import java.util.*; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Objects; import java.util.function.Function; import java.util.stream.Collectors; -import static org.junit.Assert.*; -import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.CLUSTERED_CLUSTERING_WRITER; -@RunWith(SpringRunner.class) -@ContextConfiguration(classes={BatchTestConfiguration.class}) +@ExtendWith(SpringExtension.class) +@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class,}) @TestPropertySource("classpath:backpropagation-test.properties") -public class BackPropagatedRSWriterTest { - - private static final String TEST_DB = "test-db"; - +public class BackPropagatedRSWriterTest extends MongoTestContainerHelper { private static final int TAXONOMY = 60711; private static final String ASM1 = "asm1"; @@ -76,10 +80,6 @@ public class BackPropagatedRSWriterTest { @Autowired private MongoTemplate mongoTemplate; - //Required by nosql-unit - @Autowired - private ApplicationContext applicationContext; - @Autowired private SubmittedVariantAccessioningService submittedVariantAccessioningService; @@ -87,10 +87,6 @@ public class BackPropagatedRSWriterTest { @Qualifier(CLUSTERED_CLUSTERING_WRITER) private ClusteringWriter clusteringWriter; - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName(TEST_DB).build()); - private static class RSLocus { String assembly; String contig; @@ -113,12 +109,12 @@ public String getHash() { } } - @Before + @BeforeEach public void setUp() throws Exception { mongoTemplate.getDb().drop(); } - @After + @AfterEach public void tearDown() { mongoTemplate.getDb().drop(); } @@ -146,7 +142,7 @@ public void runBackPropagationAcrossMultipleRemappings() throws AccessionCouldNo BackPropagatedRSWriter backPropagatedRSWriter = new BackPropagatedRSWriter(ASM2, this.clusteringWriter, this.submittedVariantAccessioningService, this.mongoTemplate, this.metricCompute); - backPropagatedRSWriter.write(Arrays.asList(ss2)); + backPropagatedRSWriter.write(Chunk.of(ss2)); // As of T2, see https://docs.google.com/spreadsheets/d/1KQLVCUy-vqXKgkCDt2czX6kuMfsjfCc9uBsS19MZ6dY/edit#rangeid=688023832 assertSSBackPropRSAssociation(ss2.getAccession(), rs2.getAccession(), ASM1); @@ -156,7 +152,7 @@ public void runBackPropagationAcrossMultipleRemappings() throws AccessionCouldNo this.mongoTemplate.insert(Arrays.asList(ss1_asm3_remap, ss2_asm3_remap), DbsnpSubmittedVariantEntity.class); backPropagatedRSWriter = new BackPropagatedRSWriter(ASM3, this.clusteringWriter, this.submittedVariantAccessioningService, this.mongoTemplate, this.metricCompute); - backPropagatedRSWriter.write(Arrays.asList(ss2)); + backPropagatedRSWriter.write(Chunk.of(ss2)); // As of T3, see https://docs.google.com/spreadsheets/d/1KQLVCUy-vqXKgkCDt2czX6kuMfsjfCc9uBsS19MZ6dY/edit#rangeid=490224900 assertSSBackPropRSAssociation(ss2.getAccession(), rs1.getAccession(), ASM1); @@ -169,7 +165,7 @@ public void runBackPropagationAcrossMultipleRemappings() throws AccessionCouldNo this.mongoTemplate.insert(Arrays.asList(ss1_asm4_remap, ss2_asm4_remap), DbsnpSubmittedVariantEntity.class); backPropagatedRSWriter = new BackPropagatedRSWriter(ASM4, this.clusteringWriter, this.submittedVariantAccessioningService, this.mongoTemplate, this.metricCompute); - backPropagatedRSWriter.write(Arrays.asList(ss2)); + backPropagatedRSWriter.write(Chunk.of(ss2)); // As of T4, see https://docs.google.com/spreadsheets/d/1KQLVCUy-vqXKgkCDt2czX6kuMfsjfCc9uBsS19MZ6dY/edit#rangeid=48828480 assertSSBackPropRSAssociation(ss2.getAccession(), rs3.getAccession(), ASM1); } @@ -190,7 +186,7 @@ private void assertSSBackPropRSAssociation(Long ssID, Long expectedBackPropRS, S private SubmittedVariantEntity createSS(Long ssAccession, Long rsAccession, RSLocus rsLocus, String reference, String alternate, boolean remappedFromAnotherAssembly) { - Function hashingFunction = new SubmittedVariantSummaryFunction().andThen( + Function hashingFunction = new SubmittedVariantSummaryFunction().andThen( new SHA1HashingFunction()); SubmittedVariant submittedVariant = new SubmittedVariant(rsLocus.assembly, TAXONOMY, PROJECT, rsLocus.contig, rsLocus.start, reference, alternate, rsAccession); @@ -204,7 +200,7 @@ private SubmittedVariantEntity createSS(Long ssAccession, Long rsAccession, RSLo } private ClusteredVariantEntity createRS(Long rsAccession, RSLocus rsLocus) { - Function hashingFunction = new ClusteredVariantSummaryFunction().andThen( + Function hashingFunction = new ClusteredVariantSummaryFunction().andThen( new SHA1HashingFunction()); ClusteredVariant clusteredVariant = new ClusteredVariant(rsLocus.assembly, TAXONOMY, rsLocus.contig, rsLocus.start, rsLocus.type, false, null); diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/ClusteredVariantSplittingPolicyTest.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/ClusteredVariantSplittingPolicyTest.java index ceb784278..f8a4ebc98 100644 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/ClusteredVariantSplittingPolicyTest.java +++ b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/ClusteredVariantSplittingPolicyTest.java @@ -15,9 +15,9 @@ */ package uk.ac.ebi.eva.accession.clustering.batch.io; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.springframework.test.context.junit4.SpringRunner; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.ampt2d.commons.accession.hashing.SHA1HashingFunction; import uk.ac.ebi.eva.accession.clustering.batch.io.ClusteredVariantSplittingPolicy.SplitDeterminants; @@ -29,9 +29,9 @@ import java.util.function.Function; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; -@RunWith(SpringRunner.class) +@ExtendWith(SpringExtension.class) public class ClusteredVariantSplittingPolicyTest { @Test diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/ClusteringMongoReaderRetryTest.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/ClusteringMongoReaderRetryTest.java index 2ec255230..f3c8e199e 100644 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/ClusteringMongoReaderRetryTest.java +++ b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/ClusteringMongoReaderRetryTest.java @@ -21,16 +21,16 @@ import com.mongodb.client.MongoCursor; import org.bson.Document; import org.bson.conversions.Bson; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Answers; import org.springframework.batch.item.ExecutionContext; import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.data.mongodb.core.convert.MongoConverter; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; @@ -39,8 +39,8 @@ import java.util.List; import java.util.stream.Collectors; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThrows; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; @@ -50,7 +50,7 @@ import static org.mockito.Mockito.when; import static uk.ac.ebi.eva.accession.clustering.batch.io.ClusteringMongoReaderRetryable.MAX_RETRIES; -@RunWith(SpringRunner.class) +@ExtendWith(SpringExtension.class) public class ClusteringMongoReaderRetryTest { private static final String ASSEMBLY = "GCA_000000001.1"; @@ -62,12 +62,12 @@ public class ClusteringMongoReaderRetryTest { @MockBean(answer = Answers.RETURNS_DEEP_STUBS) private MongoTemplate mongoTemplate; - @Before + @BeforeEach public void setUp(){ nonClusteredVariantReader = spy(new ClusteringMongoReaderRetryable(mongoTemplate, ASSEMBLY, CHUNK_SIZE, false)); } - @After + @AfterEach public void tearDown() { nonClusteredVariantReader.close(); } diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/ClusteringMongoReaderTest.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/ClusteringMongoReaderTest.java index 4b7dd33df..c5971393b 100644 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/ClusteringMongoReaderTest.java +++ b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/ClusteringMongoReaderTest.java @@ -15,42 +15,34 @@ */ package uk.ac.ebi.eva.accession.clustering.batch.io; -import com.lordofthejars.nosqlunit.annotation.UsingDataSet; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; -import com.mongodb.MongoClient; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.batch.item.ExecutionContext; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.ApplicationContext; +import org.springframework.core.io.ResourceLoader; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.eva.accession.clustering.test.configuration.MongoTestConfiguration; -import uk.ac.ebi.eva.accession.clustering.test.rule.FixSpringMongoDbRule; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.MongoConfiguration; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; +import uk.ac.ebi.eva.accession.core.utils.MongoTestDataLoader; import java.util.ArrayList; import java.util.List; import java.util.Objects; import static junit.framework.TestCase.assertTrue; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; -@RunWith(SpringRunner.class) +@ExtendWith(SpringExtension.class) @TestPropertySource("classpath:clustering-pipeline-test.properties") -@UsingDataSet(locations = {"/test-data/submittedVariantEntityMongoReader.json", - "/test-data/dbsnpSubmittedVariantEntityMongoReader.json"}) @ContextConfiguration(classes = {MongoConfiguration.class, MongoTestConfiguration.class}) -public class ClusteringMongoReaderTest { - - private static final String TEST_DB = "test-db"; +public class ClusteringMongoReaderTest extends MongoTestContainerHelper { private static final String ASSEMBLY = "GCA_000000001.1"; @@ -72,22 +64,20 @@ public class ClusteringMongoReaderTest { private ClusteringMongoReader nonClusteredVariantReader; - @Autowired - private MongoClient mongoClient; - @Autowired private MongoTemplate mongoTemplate; - //Required by nosql-unit @Autowired - private ApplicationContext applicationContext; + private ResourceLoader resourceLoader; + + @BeforeEach + public void setUp() { + mongoTemplate.getDb().drop(); - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName(TEST_DB).build()); + MongoTestDataLoader mongoTestDataLoader = new MongoTestDataLoader(mongoTemplate, resourceLoader); + mongoTestDataLoader.load("/test-data/submittedVariantEntityMongoReader.json"); + mongoTestDataLoader.load("/test-data/dbsnpSubmittedVariantEntityMongoReader.json"); - @Before - public void setUp(){ ExecutionContext executionContext = new ExecutionContext(); clusteredVariantReader = new ClusteringMongoReader(mongoTemplate, ASSEMBLY, CHUNK_SIZE, true); nonClusteredVariantReader = new ClusteringMongoReader(mongoTemplate, ASSEMBLY, CHUNK_SIZE, false); @@ -95,11 +85,11 @@ public void setUp(){ nonClusteredVariantReader.open(executionContext); } - @After + @AfterEach public void tearDown() { clusteredVariantReader.close(); nonClusteredVariantReader.close(); - mongoClient.dropDatabase(TEST_DB); + mongoTemplate.getDb().drop(); } @Test @@ -122,7 +112,7 @@ public void readAllSubmittedVariantsWithoutRS() { assertEquals(5, variants.size()); assertTrue(variants.stream().anyMatch(x -> Objects.equals(x.getId(), NOT_CLUSTERED_SUBMITTED_VARIANT_ID_EVA))); assertTrue(variants.stream().noneMatch(x -> Objects.equals(x.getId(), - NOT_CLUSTERED_SUBMITTED_VARIANT_ID_DBSNP))); + NOT_CLUSTERED_SUBMITTED_VARIANT_ID_DBSNP))); } private List readIntoList(ClusteringMongoReader reader) { diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/RSMergeAndSplitCandidatesReaderTest.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/RSMergeAndSplitCandidatesReaderTest.java index 8bbf4324a..6d14878c2 100644 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/RSMergeAndSplitCandidatesReaderTest.java +++ b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/RSMergeAndSplitCandidatesReaderTest.java @@ -15,45 +15,39 @@ */ package uk.ac.ebi.eva.accession.clustering.batch.io; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; -import com.mongodb.MongoClient; - -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; - +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.batch.item.ExecutionContext; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.context.ApplicationContext; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; - +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.eva.accession.clustering.configuration.batch.io.RSMergeAndSplitCandidatesReaderConfiguration; import uk.ac.ebi.eva.accession.clustering.test.configuration.BatchTestConfiguration; import uk.ac.ebi.eva.accession.clustering.test.configuration.MongoTestConfiguration; -import uk.ac.ebi.eva.accession.clustering.test.rule.FixSpringMongoDbRule; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantInactiveEntity; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantOperationEntity; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.commons.mongodb.readers.MongoDbCursorItemReader; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.RS_MERGE_CANDIDATES_READER; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.RS_SPLIT_CANDIDATES_READER; -@RunWith(SpringRunner.class) +@ExtendWith(SpringExtension.class) @TestPropertySource("classpath:clustering-pipeline-test.properties") @ContextConfiguration(classes = {RSMergeAndSplitCandidatesReaderConfiguration.class, MongoTestConfiguration.class, BatchTestConfiguration.class}) -public class RSMergeAndSplitCandidatesReaderTest { +public class RSMergeAndSplitCandidatesReaderTest extends MongoTestContainerHelper { private static final String TEST_DB = "test-db"; @@ -61,9 +55,6 @@ public class RSMergeAndSplitCandidatesReaderTest { private static final String SUBMITTED_VARIANT_OPERATION_COLLECTION = "submittedVariantOperationEntity"; - @Autowired - private MongoClient mongoClient; - @Autowired private MongoTemplate mongoTemplate; @@ -75,18 +66,49 @@ public class RSMergeAndSplitCandidatesReaderTest { @Qualifier(RS_MERGE_CANDIDATES_READER) private MongoDbCursorItemReader rsMergeCandidatesReader; - //Required by nosql-unit - @Autowired - private ApplicationContext applicationContext; + @BeforeEach + public void setUp() { + mongoTemplate.getDb().drop(); + } + + @AfterEach + public void tearDown() { + mongoTemplate.getDb().drop(); + } + + @Test + public void readAllRSSplitCandidates() throws Exception { + createSplitCandidateEntries(); + List submittedVariantOperationEntities = new ArrayList<>(); + SubmittedVariantOperationEntity submittedVariantOperationEntity; + ExecutionContext executionContext = new ExecutionContext(); + this.rsSplitCandidatesReader.open(executionContext); + while ((submittedVariantOperationEntity = rsSplitCandidatesReader.read()) != null) { + submittedVariantOperationEntities.add(submittedVariantOperationEntity); + } + //Ensure that the reader reads the two operations created during the setup routine + assertEquals(2, submittedVariantOperationEntities.size()); + } - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName(TEST_DB).build()); - private SubmittedVariantEntity createSSWithLocus(Long ssAccession, Long rsAccession, Long start, String reference, + @Test + public void readAllRSMergeCandidates() throws Exception { + createMergeCandidateEntries(); + List submittedVariantOperationEntities = new ArrayList<>(); + SubmittedVariantOperationEntity submittedVariantOperationEntity; + ExecutionContext executionContext = new ExecutionContext(); + this.rsMergeCandidatesReader.open(executionContext); + while ((submittedVariantOperationEntity = rsMergeCandidatesReader.read()) != null) { + submittedVariantOperationEntities.add(submittedVariantOperationEntity); + } + //Ensure that the reader reads the two operations created during the setup routine + assertEquals(2, submittedVariantOperationEntities.size()); + } + + private SubmittedVariantEntity createSSWithLocus(Long ssAccession, Long rsAccession, Long start, String reference, String alternate) { return new SubmittedVariantEntity(ssAccession, "hash" + ssAccession, ASSEMBLY, 60711, - "PRJ1", "chr1", start, reference, alternate, rsAccession, false, false, false, - false, 1); + "PRJ1", "chr1", start, reference, alternate, rsAccession, false, false, false, + false, 1); } private void createSplitCandidateEntries() { @@ -101,8 +123,8 @@ private void createSplitCandidateEntries() { // Note the next null in accessionIdDestiny. We are not merging the submitted variant into // anything. We are updating the submitted variant, changing its rs field splitOperation1.fill(RSMergeAndSplitCandidatesReaderConfiguration.SPLIT_CANDIDATES_EVENT_TYPE, - ss1.getAccession(), null, "Hash mismatch with rs1", - Arrays.asList(ss1, ss2, ss3)); + ss1.getAccession(), null, "Hash mismatch with rs1", + Arrays.asList(ss1, ss2, ss3)); splitOperation1.setId(ClusteringWriter.getSplitCandidateId(splitOperation1)); SubmittedVariantInactiveEntity ss4 = new SubmittedVariantInactiveEntity( @@ -115,9 +137,9 @@ private void createSplitCandidateEntries() { // Note the next null in accessionIdDestiny. We are not merging the submitted variant into // anything. We are updating the submitted variant, changing its rs field splitOperation2.fill(RSMergeAndSplitCandidatesReaderConfiguration.SPLIT_CANDIDATES_EVENT_TYPE, - ss4.getAccession(), - null, "Hash mismatch with rs2", - Arrays.asList(ss4, ss5, ss6)); + ss4.getAccession(), + null, "Hash mismatch with rs2", + Arrays.asList(ss4, ss5, ss6)); splitOperation2.setId(ClusteringWriter.getSplitCandidateId(splitOperation2)); mongoTemplate.insert(splitOperation1, SUBMITTED_VARIANT_OPERATION_COLLECTION); @@ -136,8 +158,8 @@ private void createMergeCandidateEntries() { // Note the next null in accessionIdDestiny. We are not merging the submitted variant into // anything. We are updating the submitted variant, changing its rs field mergeOperation1.fill(RSMergeAndSplitCandidatesReaderConfiguration.MERGE_CANDIDATES_EVENT_TYPE, - ss1.getAccession(), null, "Different RS with matching loci", - Arrays.asList(ss1, ss2, ss3)); + ss1.getAccession(), null, "Different RS with matching loci", + Arrays.asList(ss1, ss2, ss3)); mergeOperation1.setId(ClusteringWriter.getMergeCandidateId(mergeOperation1)); SubmittedVariantInactiveEntity ss4 = new SubmittedVariantInactiveEntity( @@ -150,54 +172,12 @@ private void createMergeCandidateEntries() { // Note the next null in accessionIdDestiny. We are not merging the submitted variant into // anything. We are updating the submitted variant, changing its rs field mergeOperation2.fill(RSMergeAndSplitCandidatesReaderConfiguration.MERGE_CANDIDATES_EVENT_TYPE, - ss4.getAccession(), - null, "Different RS with matching loci", - Arrays.asList(ss4, ss5, ss6)); + ss4.getAccession(), + null, "Different RS with matching loci", + Arrays.asList(ss4, ss5, ss6)); mergeOperation2.setId(ClusteringWriter.getMergeCandidateId(mergeOperation2)); mongoTemplate.insert(mergeOperation1, SUBMITTED_VARIANT_OPERATION_COLLECTION); mongoTemplate.insert(mergeOperation2, SUBMITTED_VARIANT_OPERATION_COLLECTION); } - - @Before - public void setUp() { - cleanupDB(); - } - - @After - public void tearDown() { - cleanupDB(); - } - - private void cleanupDB() { - mongoClient.dropDatabase(TEST_DB); - } - - @Test - public void readAllRSSplitCandidates() throws Exception { - createSplitCandidateEntries(); - List submittedVariantOperationEntities = new ArrayList<>(); - SubmittedVariantOperationEntity submittedVariantOperationEntity; - ExecutionContext executionContext = new ExecutionContext(); - this.rsSplitCandidatesReader.open(executionContext); - while ((submittedVariantOperationEntity = rsSplitCandidatesReader.read()) != null) { - submittedVariantOperationEntities.add(submittedVariantOperationEntity); - } - //Ensure that the reader reads the two operations created during the setup routine - assertEquals(2, submittedVariantOperationEntities.size()); - } - - @Test - public void readAllRSMergeCandidates() throws Exception { - createMergeCandidateEntries(); - List submittedVariantOperationEntities = new ArrayList<>(); - SubmittedVariantOperationEntity submittedVariantOperationEntity; - ExecutionContext executionContext = new ExecutionContext(); - this.rsMergeCandidatesReader.open(executionContext); - while ((submittedVariantOperationEntity = rsMergeCandidatesReader.read()) != null) { - submittedVariantOperationEntities.add(submittedVariantOperationEntity); - } - //Ensure that the reader reads the two operations created during the setup routine - assertEquals(2, submittedVariantOperationEntities.size()); - } } \ No newline at end of file diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/RSMergeWriterTest.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/RSMergeWriterTest.java index 7533077fe..c18812865 100644 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/RSMergeWriterTest.java +++ b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/RSMergeWriterTest.java @@ -15,40 +15,32 @@ */ package uk.ac.ebi.eva.accession.clustering.batch.io; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; -import com.mongodb.MongoClient; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.batch.item.Chunk; import org.springframework.batch.item.ExecutionContext; import org.springframework.batch.item.ItemWriter; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.context.ApplicationContext; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.data.mongodb.core.query.Criteria; import org.springframework.data.mongodb.core.query.Query; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; -import uk.ac.ebi.ampt2d.commons.accession.core.exceptions.AccessionDeprecatedException; -import uk.ac.ebi.ampt2d.commons.accession.core.exceptions.AccessionDoesNotExistException; +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.ampt2d.commons.accession.core.exceptions.AccessionMergedException; import uk.ac.ebi.ampt2d.commons.accession.core.models.EventType; import uk.ac.ebi.ampt2d.commons.accession.hashing.SHA1HashingFunction; import uk.ac.ebi.ampt2d.commons.accession.persistence.mongodb.document.InactiveSubDocument; - import uk.ac.ebi.eva.accession.clustering.configuration.batch.io.RSMergeAndSplitCandidatesReaderConfiguration; import uk.ac.ebi.eva.accession.clustering.configuration.batch.io.RSMergeAndSplitWriterConfiguration; import uk.ac.ebi.eva.accession.clustering.metric.ClusteringMetric; import uk.ac.ebi.eva.accession.clustering.test.DatabaseState; import uk.ac.ebi.eva.accession.clustering.test.configuration.BatchTestConfiguration; import uk.ac.ebi.eva.accession.clustering.test.configuration.MongoTestConfiguration; -import uk.ac.ebi.eva.accession.clustering.test.rule.FixSpringMongoDbRule; import uk.ac.ebi.eva.accession.core.model.ClusteredVariant; import uk.ac.ebi.eva.accession.core.model.IClusteredVariant; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpClusteredVariantEntity; @@ -62,6 +54,7 @@ import uk.ac.ebi.eva.accession.core.service.nonhuman.ClusteredVariantAccessioningService; import uk.ac.ebi.eva.accession.core.service.nonhuman.SubmittedVariantAccessioningService; import uk.ac.ebi.eva.accession.core.summary.ClusteredVariantSummaryFunction; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.commons.core.models.pipeline.Variant; import uk.ac.ebi.eva.commons.mongodb.readers.MongoDbCursorItemReader; import uk.ac.ebi.eva.metrics.metric.MetricCompute; @@ -74,22 +67,20 @@ import java.util.stream.Collectors; import java.util.stream.Stream; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertThrows; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.CLUSTERED_CLUSTERING_WRITER; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.RS_MERGE_CANDIDATES_READER; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.RS_MERGE_WRITER; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.RS_SPLIT_CANDIDATES_READER; -@RunWith(SpringRunner.class) +@ExtendWith(SpringExtension.class) @TestPropertySource("classpath:merge-split-test.properties") @ContextConfiguration(classes = {RSMergeAndSplitCandidatesReaderConfiguration.class, RSMergeAndSplitWriterConfiguration.class, MongoTestConfiguration.class, BatchTestConfiguration.class}) -public class RSMergeWriterTest { - - private static final String TEST_DB = "test-db"; +public class RSMergeWriterTest extends MongoTestContainerHelper { private static final String ASSEMBLY = "GCA_000000001.1"; @@ -99,9 +90,6 @@ public class RSMergeWriterTest { private static final String SUBMITTED_VARIANT_OPERATION_COLLECTION = "submittedVariantOperationEntity"; - @Autowired - private MongoClient mongoClient; - @Autowired private MongoTemplate mongoTemplate; @@ -121,14 +109,6 @@ public class RSMergeWriterTest { @Qualifier(CLUSTERED_CLUSTERING_WRITER) private ClusteringWriter clusteringWriter; - //Required by nosql-unit - @Autowired - private ApplicationContext applicationContext; - - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName(TEST_DB).build()); - private SubmittedVariantEntity ss1, ss2, ss3, ss4, ss5, ss6, ss7, ss8, ss9; @Autowired @@ -156,8 +136,8 @@ private SubmittedVariantEntity createSS(Long ssAccession, Long rsAccession, Long String alternate, String assemblyToUse) { return new SubmittedVariantEntity(ssAccession, "hash" + ssAccession + assemblyToUse, assemblyToUse, 60711, - "PRJ1", "chr1", start, reference, alternate, rsAccession, false, false, false, - false, 1); + "PRJ1", "chr1", start, reference, alternate, rsAccession, false, false, false, + false, 1); } private void createMergeAndSplitCandidateEntries() { @@ -187,8 +167,8 @@ private void createMergeAndSplitCandidateEntries() { SubmittedVariantInactiveEntity ssInactive4 = new SubmittedVariantInactiveEntity(ss4); SubmittedVariantOperationEntity mergeOperation1 = new SubmittedVariantOperationEntity(); mergeOperation1.fill(RSMergeAndSplitCandidatesReaderConfiguration.MERGE_CANDIDATES_EVENT_TYPE, - ss1.getAccession(), null, "Different RS with matching loci", - Arrays.asList(ssInactive1, ssInactive4)); + ss1.getAccession(), null, "Different RS with matching loci", + Arrays.asList(ssInactive1, ssInactive4)); mergeOperation1.setId(ClusteringWriter.getMergeCandidateId(mergeOperation1)); //ss2 will be inserted to dbsnpSubmittedVariantEntity and ss5 to submittedVariantEntity collections respectively @@ -199,9 +179,9 @@ private void createMergeAndSplitCandidateEntries() { SubmittedVariantInactiveEntity ssInactive5 = new SubmittedVariantInactiveEntity(ss5); SubmittedVariantOperationEntity mergeOperation2 = new SubmittedVariantOperationEntity(); mergeOperation2.fill(RSMergeAndSplitCandidatesReaderConfiguration.MERGE_CANDIDATES_EVENT_TYPE, - ss2.getAccession(), - null, "Different RS with matching loci", - Arrays.asList(ssInactive2, ssInactive5)); + ss2.getAccession(), + null, "Different RS with matching loci", + Arrays.asList(ssInactive2, ssInactive5)); mergeOperation2.setId(ClusteringWriter.getMergeCandidateId(mergeOperation2)); //Candidates for split are entries with same RS but different locus @@ -218,16 +198,16 @@ private void createMergeAndSplitCandidateEntries() { SubmittedVariantOperationEntity splitOperation2 = new SubmittedVariantOperationEntity(); SubmittedVariantOperationEntity splitOperation3 = new SubmittedVariantOperationEntity(); splitOperation1.fill(RSMergeAndSplitCandidatesReaderConfiguration.SPLIT_CANDIDATES_EVENT_TYPE, - ss2.getAccession(), "Hash mismatch with " + ss2.getClusteredVariantAccession(), - Arrays.asList(ssInactive2, ssInactive8)); + ss2.getAccession(), "Hash mismatch with " + ss2.getClusteredVariantAccession(), + Arrays.asList(ssInactive2, ssInactive8)); splitOperation1.setId(ClusteringWriter.getSplitCandidateId(splitOperation1)); splitOperation2.fill(RSMergeAndSplitCandidatesReaderConfiguration.SPLIT_CANDIDATES_EVENT_TYPE, - ss4.getAccession(), "Hash mismatch with " + ss4.getClusteredVariantAccession(), - Arrays.asList(ssInactive4, ssInactive6, ssInactive7)); + ss4.getAccession(), "Hash mismatch with " + ss4.getClusteredVariantAccession(), + Arrays.asList(ssInactive4, ssInactive6, ssInactive7)); splitOperation2.setId(ClusteringWriter.getSplitCandidateId(splitOperation2)); splitOperation3.fill(RSMergeAndSplitCandidatesReaderConfiguration.SPLIT_CANDIDATES_EVENT_TYPE, - ss5.getAccession(), "Hash mismatch with " + ss5.getClusteredVariantAccession(), - Arrays.asList(ssInactive5, ssInactive9)); + ss5.getAccession(), "Hash mismatch with " + ss5.getClusteredVariantAccession(), + Arrays.asList(ssInactive5, ssInactive9)); splitOperation3.setId(ClusteringWriter.getSplitCandidateId(splitOperation3)); List ssToInsertToDbsnpSVE = Arrays.asList(ss1, ss2); @@ -236,33 +216,29 @@ private void createMergeAndSplitCandidateEntries() { mongoTemplate.insert(ssToInsertToDbsnpSVE, DBSNP_SUBMITTED_VARIANT_COLLECTION); mongoTemplate.insert(ssToInsertToSVE, SUBMITTED_VARIANT_COLLECTION); mongoTemplate.insert(Arrays.asList(mergeOperation1, mergeOperation2, - splitOperation1, splitOperation2, splitOperation3), - SUBMITTED_VARIANT_OPERATION_COLLECTION); - } - - private void cleanup() { - mongoClient.dropDatabase(TEST_DB); - metricCompute.clearCount(); + splitOperation1, splitOperation2, splitOperation3), + SUBMITTED_VARIANT_OPERATION_COLLECTION); } - @Before + @BeforeEach public void setUp() { - cleanup(); + mongoTemplate.getDb().drop(); + metricCompute.clearCount(); } - @After + @AfterEach public void tearDown() { - cleanup(); + mongoTemplate.getDb().drop(); + metricCompute.clearCount(); } - private void assertRSAssociatedWithSS(Long rsAccession, SubmittedVariantEntity submittedVariantEntity) - throws AccessionDoesNotExistException, AccessionMergedException, AccessionDeprecatedException { + private void assertRSAssociatedWithSS(Long rsAccession, SubmittedVariantEntity submittedVariantEntity) { assertEquals(rsAccession, - submittedVariantAccessioningService - .getAllActiveByAssemblyAndAccessionIn(ASSEMBLY, - Collections.singletonList( - submittedVariantEntity.getAccession())) - .stream().findFirst().get().getData().getClusteredVariantAccession()); + submittedVariantAccessioningService + .getAllActiveByAssemblyAndAccessionIn(ASSEMBLY, + Collections.singletonList( + submittedVariantEntity.getAccession())) + .stream().findFirst().get().getData().getClusteredVariantAccession()); } @Test @@ -294,24 +270,28 @@ public void testWriteRSMerges() throws Exception { } //Perform merge - rsMergeWriter.write(submittedVariantOperationEntities); + rsMergeWriter.write(new Chunk<>(submittedVariantOperationEntities)); //Ensure operations collections have appropriate entries to reflect rs4 merged to rs1 & rs5 merged to rs2 assertEquals(Long.valueOf(1L), clusteredVariantOperationRepository.findAllByAccession(4L).get(0) - .getMergedInto()); + .getMergedInto()); assertEquals(Long.valueOf(2L), clusteredVariantOperationRepository.findAllByAccession(5L).get(0) - .getMergedInto()); - assertEquals ("Original rs4 associated with SS was merged into rs1.", - submittedVariantOperationRepository.findAllByAccession(4L).get(0).getReason()); + .getMergedInto()); + assertEquals("Original rs4 associated with SS was merged into rs1.", + submittedVariantOperationRepository.findAllByAccession(4L).get(0).getReason()); assertEquals("Original rs5 associated with SS was merged into rs2.", - submittedVariantOperationRepository.findAllByAccession(5L).get(0).getReason()); + submittedVariantOperationRepository.findAllByAccession(5L).get(0).getReason()); // Ensure that RS1 and RS2 are present in the clustered variant collection post merge assertEquals(Long.valueOf(1L), clusteredVariantAccessioningService.getByAccession(1L).getAccession()); assertEquals(Long.valueOf(2L), clusteredVariantAccessioningService.getByAccession(2L).getAccession()); - assertThrows(AccessionMergedException.class, () -> {clusteredVariantAccessioningService.getByAccession(4L);}); - assertThrows(AccessionMergedException.class, () -> {clusteredVariantAccessioningService.getByAccession(5L);}); + assertThrows(AccessionMergedException.class, () -> { + clusteredVariantAccessioningService.getByAccession(4L); + }); + assertThrows(AccessionMergedException.class, () -> { + clusteredVariantAccessioningService.getByAccession(5L); + }); //After merge SS-RS associations: rs4 merged to rs1 & rs5 merged to rs2 /* @@ -337,13 +317,13 @@ public void testWriteRSMerges() throws Exception { // After rs4 merge to rs1 and rs5 merge to rs2 // split candidates operations involving rs4 and rs5 should no longer be present assertFalse(submittedVariantOperationRepository - .findAllByAccession(4L).stream() - .anyMatch(event -> event.getEventType().equals( - RSMergeAndSplitCandidatesReaderConfiguration.SPLIT_CANDIDATES_EVENT_TYPE))); + .findAllByAccession(4L).stream() + .anyMatch(event -> event.getEventType().equals( + RSMergeAndSplitCandidatesReaderConfiguration.SPLIT_CANDIDATES_EVENT_TYPE))); assertFalse(submittedVariantOperationRepository - .findAllByAccession(5L).stream() - .anyMatch(event -> event.getEventType().equals( - RSMergeAndSplitCandidatesReaderConfiguration.SPLIT_CANDIDATES_EVENT_TYPE))); + .findAllByAccession(5L).stream() + .anyMatch(event -> event.getEventType().equals( + RSMergeAndSplitCandidatesReaderConfiguration.SPLIT_CANDIDATES_EVENT_TYPE))); // Post-merge SS1, SS4, SS6 and SS7 all share rs1 but have different locus // and hence generate one split candidate event // Post-merge SS2, SS5, SS8 and SS9 all share rs2 but have different locus @@ -352,7 +332,7 @@ public void testWriteRSMerges() throws Exception { SubmittedVariantOperationEntity tempObj; rsSplitCandidatesReader.open(new ExecutionContext()); while ((tempObj = rsSplitCandidatesReader.read()) != null) { - splitEvents.add(tempObj); + splitEvents.add(tempObj); } assertSplitCandidateEvents(1L, Arrays.asList(1L, 4L, 6L, 7L), splitEvents); assertSplitCandidateEvents(2L, Arrays.asList(2L, 5L, 8L, 9L), splitEvents); @@ -369,10 +349,10 @@ public void testWriteRSMerges() throws Exception { // Check if a given RS has the participating SS listed in the split candidates event private void assertSplitCandidateEvents(Long rsAccession, List expectedParticipatingSS, - List splitEvents) throws Exception { + List splitEvents) { splitEvents = splitEvents.stream().filter(e -> e.getInactiveObjects().get(0) - .getClusteredVariantAccession().equals(rsAccession)) - .collect(Collectors.toList()); + .getClusteredVariantAccession().equals(rsAccession)) + .collect(Collectors.toList()); assertEquals(1, splitEvents.size()); SubmittedVariantOperationEntity splitEvent = splitEvents.get(0); assertEquals("Hash mismatch with " + rsAccession, splitEvent.getReason()); @@ -380,12 +360,12 @@ private void assertSplitCandidateEvents(Long rsAccession, List expectedPar assertEquals(expectedParticipatingSS.size(), participatingSSInSplitEvent.size()); assertTrue(participatingSSInSplitEvent.stream() - .map(InactiveSubDocument::getAccession) - .collect(Collectors.toSet()) - .containsAll(expectedParticipatingSS)); + .map(InactiveSubDocument::getAccession) + .collect(Collectors.toSet()) + .containsAll(expectedParticipatingSS)); // Ensure that all the inactive objects have the same RS assertFalse(participatingSSInSplitEvent.stream() - .anyMatch(e -> !e.getClusteredVariantAccession().equals(rsAccession))); + .anyMatch(e -> !e.getClusteredVariantAccession().equals(rsAccession))); } @Test @@ -411,7 +391,7 @@ public void testMultiLevelRSMerges() throws Exception { } //Perform merge - rsMergeWriter.write(submittedVariantOperationEntities); + rsMergeWriter.write(new Chunk<>(submittedVariantOperationEntities)); //After merge SS-RS associations: rs2 merged to rs1, Merge target for rs3 inferred as: rs3 -> rs2 -> rs1 /* @@ -458,7 +438,7 @@ public void testRSWithSameHashAsMergeDestinationAlreadyExistsInDBWithHigherAcces } //Perform merge - rsMergeWriter.write(submittedVariantOperationEntities); + rsMergeWriter.write(new Chunk<>(submittedVariantOperationEntities)); // Check rs2,rs3 merged into to rs1 assertMergeOp(ss2.getClusteredVariantAccession(), ss1.getClusteredVariantAccession(), ASSEMBLY); @@ -496,7 +476,7 @@ public void testRSWithSameHashAsMergeDestinationAlreadyExistsInDBWithLowerAccess } //Perform merge - rsMergeWriter.write(submittedVariantOperationEntities); + rsMergeWriter.write(new Chunk<>(submittedVariantOperationEntities)); // Check rs2,rs3 merged into to rs1 assertMergeOp(ss2.getClusteredVariantAccession(), ss1.getClusteredVariantAccession(), ASSEMBLY); @@ -512,9 +492,9 @@ private void assertMergeOp(Long mergee, Long merger, String assemblyToUse) { assertEquals(0, this.clusteredVariantAccessioningService .getAllActiveByAssemblyAndAccessionIn(assemblyToUse, Arrays.asList(mergee)).size()); assertEquals(1, this.mongoTemplate.find(Query.query(Criteria.where("inactiveObjects.asm") - .is(assemblyToUse).and("accession").is(mergee) - .and("mergeInto").is(merger)), - DbsnpClusteredVariantOperationEntity.class).size()); + .is(assemblyToUse).and("accession").is(mergee) + .and("mergeInto").is(merger)), + DbsnpClusteredVariantOperationEntity.class).size()); } public void createMultiLevelMergeScenario() { @@ -528,23 +508,23 @@ public void createMultiLevelMergeScenario() { SubmittedVariantOperationEntity mergeOperation1 = new SubmittedVariantOperationEntity(); mergeOperation1.fill(RSMergeAndSplitCandidatesReaderConfiguration.MERGE_CANDIDATES_EVENT_TYPE, - ss1.getAccession(), null, "Different RS with matching loci", - Stream.of(ss1, ss2).map(SubmittedVariantInactiveEntity::new).collect(Collectors.toList())); + ss1.getAccession(), null, "Different RS with matching loci", + Stream.of(ss1, ss2).map(SubmittedVariantInactiveEntity::new).collect(Collectors.toList())); mergeOperation1.setId(ClusteringWriter.getMergeCandidateId(mergeOperation1)); SubmittedVariantOperationEntity mergeOperation2 = new SubmittedVariantOperationEntity(); mergeOperation2.fill(RSMergeAndSplitCandidatesReaderConfiguration.MERGE_CANDIDATES_EVENT_TYPE, - ss3.getAccession(), null, "Different RS with matching loci", - Stream.of(ss3, ss4).map(SubmittedVariantInactiveEntity::new).collect(Collectors.toList())); + ss3.getAccession(), null, "Different RS with matching loci", + Stream.of(ss3, ss4).map(SubmittedVariantInactiveEntity::new).collect(Collectors.toList())); mergeOperation2.setId(ClusteringWriter.getMergeCandidateId(mergeOperation2)); SubmittedVariantOperationEntity splitOperation1 = new SubmittedVariantOperationEntity(); splitOperation1.fill(RSMergeAndSplitCandidatesReaderConfiguration.SPLIT_CANDIDATES_EVENT_TYPE, - ss2.getAccession(), null, "Hash mismatch with " + ss2.getClusteredVariantAccession(), - Stream.of(ss2, ss3).map(SubmittedVariantInactiveEntity::new).collect(Collectors.toList())); + ss2.getAccession(), null, "Hash mismatch with " + ss2.getClusteredVariantAccession(), + Stream.of(ss2, ss3).map(SubmittedVariantInactiveEntity::new).collect(Collectors.toList())); splitOperation1.setId(ClusteringWriter.getSplitCandidateId(splitOperation1)); this.mongoTemplate.insert(Arrays.asList(mergeOperation1, mergeOperation2, splitOperation1), - SUBMITTED_VARIANT_OPERATION_COLLECTION); + SUBMITTED_VARIANT_OPERATION_COLLECTION); } private void createSimilarEntriesInAnotherAssembly() { @@ -555,7 +535,7 @@ private void createSimilarEntriesInAnotherAssembly() { SubmittedVariantEntity ss3_another_asm = createSS(3L, 2L, 103L, "A", "G", "ASM_ANOTHER"); SubmittedVariantEntity ss4_another_asm = createSS(4L, 3L, 103L, "T", "C", "ASM_ANOTHER"); this.mongoTemplate.insert(Arrays.asList(ss1_another_asm, ss2_another_asm, ss3_another_asm), - DBSNP_SUBMITTED_VARIANT_COLLECTION); + DBSNP_SUBMITTED_VARIANT_COLLECTION); this.mongoTemplate.insert(Collections.singletonList(ss4_another_asm), SUBMITTED_VARIANT_COLLECTION); DbsnpClusteredVariantEntity rs1_another_asm = this.createRS(ss1_another_asm); @@ -563,8 +543,8 @@ private void createSimilarEntriesInAnotherAssembly() { // see https://www.ebi.ac.uk/panda/jira/browse/EVA-2974?focusedCommentId=405919&page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel#comment-405919 DbsnpClusteredVariantOperationEntity rs2ToRs1MergeOp = new DbsnpClusteredVariantOperationEntity(); rs2ToRs1MergeOp.fill(EventType.MERGED, 2L, 1L, - "Identical clustered variant received multiple RS identifiers", - Arrays.asList(new DbsnpClusteredVariantInactiveEntity(rs1_another_asm))); + "Identical clustered variant received multiple RS identifiers", + Arrays.asList(new DbsnpClusteredVariantInactiveEntity(rs1_another_asm))); this.mongoTemplate.save(rs2ToRs1MergeOp); } @@ -589,11 +569,11 @@ public void testIdempotentMergeWrites() throws Exception { } //First run of RS Merges - rsMergeWriter.write(submittedVariantOperationEntities); + rsMergeWriter.write(new Chunk<>(submittedVariantOperationEntities)); DatabaseState databaseStateAfterFirstMergeWrite = DatabaseState.getCurrentDatabaseState(this.mongoTemplate); - rsMergeWriter.write(submittedVariantOperationEntities); + rsMergeWriter.write(new Chunk<>(submittedVariantOperationEntities)); DatabaseState databaseStateAfterSecondMergeWrite = DatabaseState.getCurrentDatabaseState(this.mongoTemplate); - rsMergeWriter.write(submittedVariantOperationEntities); + rsMergeWriter.write(new Chunk<>(submittedVariantOperationEntities)); DatabaseState databaseStateAfterThirdMergeWrite = DatabaseState.getCurrentDatabaseState(this.mongoTemplate); assertEquals(databaseStateAfterSecondMergeWrite, databaseStateAfterFirstMergeWrite); @@ -601,15 +581,15 @@ public void testIdempotentMergeWrites() throws Exception { } private DbsnpClusteredVariantEntity createRS(SubmittedVariantEntity sve) { - Function hashingFunction = new ClusteredVariantSummaryFunction().andThen( + Function hashingFunction = new ClusteredVariantSummaryFunction().andThen( new SHA1HashingFunction()); ClusteredVariant cv = new ClusteredVariant(sve.getReferenceSequenceAccession(), sve.getTaxonomyAccession(), - sve.getContig(), - sve.getStart(), - new Variant(sve.getContig(), sve.getStart(), sve.getStart(), - sve.getReferenceAllele(), - sve.getAlternateAllele()).getType(), - true, null); + sve.getContig(), + sve.getStart(), + new Variant(sve.getContig(), sve.getStart(), sve.getStart(), + sve.getReferenceAllele(), + sve.getAlternateAllele()).getType(), + true, null); String hash = hashingFunction.apply(cv); return new DbsnpClusteredVariantEntity(sve.getClusteredVariantAccession(), hash, cv); } diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/RSSplitWriterTest.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/RSSplitWriterTest.java index 29b6054f8..e2c326179 100644 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/RSSplitWriterTest.java +++ b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/RSSplitWriterTest.java @@ -15,37 +15,29 @@ */ package uk.ac.ebi.eva.accession.clustering.batch.io; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; -import com.mongodb.MongoClient; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; -import org.junit.runner.RunWith; - +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mockito; import org.springframework.batch.core.JobExecution; +import org.springframework.batch.item.Chunk; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.test.mock.mockito.MockBean; -import org.springframework.context.ApplicationContext; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.ampt2d.commons.accession.core.models.AccessionWrapper; import uk.ac.ebi.ampt2d.commons.accession.hashing.SHA1HashingFunction; - import uk.ac.ebi.eva.accession.clustering.configuration.batch.io.RSMergeAndSplitCandidatesReaderConfiguration; import uk.ac.ebi.eva.accession.clustering.configuration.batch.io.RSMergeAndSplitWriterConfiguration; import uk.ac.ebi.eva.accession.clustering.metric.ClusteringMetric; import uk.ac.ebi.eva.accession.clustering.test.DatabaseState; import uk.ac.ebi.eva.accession.clustering.test.configuration.BatchTestConfiguration; import uk.ac.ebi.eva.accession.clustering.test.configuration.MongoTestConfiguration; -import uk.ac.ebi.eva.accession.clustering.test.rule.FixSpringMongoDbRule; import uk.ac.ebi.eva.accession.core.model.ISubmittedVariant; import uk.ac.ebi.eva.accession.core.model.SubmittedVariant; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpClusteredVariantEntity; @@ -56,6 +48,7 @@ import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantOperationEntity; import uk.ac.ebi.eva.accession.core.service.nonhuman.SubmittedVariantAccessioningService; import uk.ac.ebi.eva.accession.core.summary.SubmittedVariantSummaryFunction; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.metrics.metric.MetricCompute; import java.io.File; @@ -68,28 +61,20 @@ import java.util.stream.Collectors; import java.util.stream.Stream; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.CLUSTERED_CLUSTERING_WRITER; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.RS_SPLIT_WRITER; -@RunWith(SpringRunner.class) +@ExtendWith(SpringExtension.class) @TestPropertySource("classpath:merge-split-test.properties") -@ContextConfiguration(classes = {RSMergeAndSplitCandidatesReaderConfiguration.class, RSMergeAndSplitWriterConfiguration.class, - MongoTestConfiguration.class, BatchTestConfiguration.class}) -public class RSSplitWriterTest { - - private static final String TEST_DB = "test-db"; +@ContextConfiguration(classes = {RSMergeAndSplitCandidatesReaderConfiguration.class, + RSMergeAndSplitWriterConfiguration.class, MongoTestConfiguration.class, BatchTestConfiguration.class}) +public class RSSplitWriterTest extends MongoTestContainerHelper { private static final String ASSEMBLY = "GCA_000000001.1"; - @Rule - public final ExpectedException thrown = ExpectedException.none(); - - @Autowired - private MongoClient mongoClient; - @Autowired private MongoTemplate mongoTemplate; @@ -107,14 +92,6 @@ public class RSSplitWriterTest { @MockBean private JobExecution jobExecution; - //Required by nosql-unit - @Autowired - private ApplicationContext applicationContext; - - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName(TEST_DB).build()); - private SubmittedVariantEntity ss1, ss2, ss3, ss4, ss5, ss6; @Autowired @@ -123,22 +100,20 @@ public class RSSplitWriterTest { @Autowired private MetricCompute metricCompute; - @Before + @BeforeEach public void setUp() throws IOException { - cleanup(); + mongoTemplate.getDb().drop(); + metricCompute.clearCount(); + Files.deleteIfExists(this.rsReportFile.toPath()); Mockito.when(jobExecution.getJobId()).thenReturn(1L); rsSplitWriter.setJobExecution(jobExecution); clusteringWriter.setJobExecution(jobExecution); } - @After + @AfterEach public void tearDown() throws IOException { - cleanup(); - } - - private void cleanup() throws IOException { - mongoClient.dropDatabase(TEST_DB); + mongoTemplate.getDb().drop(); metricCompute.clearCount(); Files.deleteIfExists(this.rsReportFile.toPath()); } @@ -148,7 +123,7 @@ private SubmittedVariantEntity createSS(Long ssAccession, Long rsAccession, Long Function hashingFunction = new SubmittedVariantSummaryFunction().andThen( new SHA1HashingFunction()); SubmittedVariant temp = new SubmittedVariant(ASSEMBLY, 60711, "PRJ1", "chr1", start, reference, alternate, - rsAccession); + rsAccession); return new SubmittedVariantEntity(ssAccession, hashingFunction.apply(temp), temp, 1); } @@ -163,29 +138,29 @@ public void testSplitsWithUnequalNumberOfDistinctHashes() throws Exception { // Create multiple entries in clustered variant collection - one for each distinct loci but with the same accession List multipleEntriesWithSameRS = Stream.of(ss1, ss2, ss4) - .map(clusteringWriter::toClusteredVariantEntity) - .collect(Collectors.toList()); + .map(clusteringWriter::toClusteredVariantEntity) + .collect(Collectors.toList()); mongoTemplate.insert(Arrays.asList(ss1, ss2, ss3), DbsnpSubmittedVariantEntity.class); mongoTemplate.insert(Collections.singletonList(ss4), SubmittedVariantEntity.class); mongoTemplate.insert(multipleEntriesWithSameRS, DbsnpClusteredVariantEntity.class); SubmittedVariantOperationEntity splitOperation = new SubmittedVariantOperationEntity(); splitOperation.fill(RSMergeAndSplitCandidatesReaderConfiguration.SPLIT_CANDIDATES_EVENT_TYPE, - ss1.getAccession(), "Hash mismatch with " + ss1.getClusteredVariantAccession(), - Stream.of(ss1, ss2, ss3, ss4).map(SubmittedVariantInactiveEntity::new) - .collect(Collectors.toList()) + ss1.getAccession(), "Hash mismatch with " + ss1.getClusteredVariantAccession(), + Stream.of(ss1, ss2, ss3, ss4).map(SubmittedVariantInactiveEntity::new) + .collect(Collectors.toList()) ); mongoTemplate.insert(Collections.singletonList(splitOperation), SubmittedVariantOperationEntity.class); - rsSplitWriter.write(Collections.singletonList(splitOperation)); + rsSplitWriter.write(Chunk.of(splitOperation)); // Per splitting policy, ensure that SS2 and SS3 get to retain the old RS // because the RS hash for start position 101 is supported in these 2 variants List> ssAssociatedWithRS1 = - submittedVariantAccessioningService.getByClusteredVariantAccessionIn( - Collections.singletonList(sameRSAccessionToBeUsedForDifferentLoci)); + submittedVariantAccessioningService.getByClusteredVariantAccessionIn( + Collections.singletonList(sameRSAccessionToBeUsedForDifferentLoci)); assertEquals(2, ssAssociatedWithRS1.size()); assertTrue(ssAssociatedWithRS1.stream().map(AccessionWrapper::getAccession).collect(Collectors.toSet()) - .containsAll(Arrays.asList(ss2.getAccession(), ss3.getAccession()))); + .containsAll(Arrays.asList(ss2.getAccession(), ss3.getAccession()))); /* Check clustering counts */ // 2 new RS IDs created for hashes with start positions 100 and 102 assertEquals(2, metricCompute.getCount(ClusteringMetric.CLUSTERED_VARIANTS_CREATED)); @@ -200,9 +175,9 @@ public void testSplitsWithUnequalNumberOfDistinctHashes() throws Exception { // RS ID created for a given hash in not deterministic during the split. // So, use regex to check for just the hashes in the RS report. assertTrue(rsReportLines.stream() - .anyMatch(line -> line.matches(".*\t20610F413155992F45A41FB1DC6005B2FAA1565B"))); + .anyMatch(line -> line.matches(".*\t20610F413155992F45A41FB1DC6005B2FAA1565B"))); assertTrue(rsReportLines.stream() - .anyMatch(line -> line.matches(".*\t717DE940ADEB9E9DCAF09600065BAAF9A91EE713"))); + .anyMatch(line -> line.matches(".*\t717DE940ADEB9E9DCAF09600065BAAF9A91EE713"))); } @Test @@ -218,20 +193,20 @@ public void testSplitsWithEqualNumberOfDistinctHashes() throws Exception { // Create multiple entries in clustered variant collection - one for each distinct loci but with the same accession List multipleEntriesWithSameRS = Stream.of(ss1, ss3, ss5) - .map(clusteringWriter::toClusteredVariantEntity) - .collect(Collectors.toList()); + .map(clusteringWriter::toClusteredVariantEntity) + .collect(Collectors.toList()); mongoTemplate.insert(Arrays.asList(ss1, ss2, ss3), DbsnpSubmittedVariantEntity.class); mongoTemplate.insert(Arrays.asList(ss4, ss5, ss6), SubmittedVariantEntity.class); mongoTemplate.insert(multipleEntriesWithSameRS, DbsnpClusteredVariantEntity.class); SubmittedVariantOperationEntity splitOperation = new SubmittedVariantOperationEntity(); splitOperation.fill(RSMergeAndSplitCandidatesReaderConfiguration.SPLIT_CANDIDATES_EVENT_TYPE, - ss1.getAccession(), "Hash mismatch with " + ss1.getClusteredVariantAccession(), - Stream.of(ss1, ss2, ss3, ss4, ss5, ss6).map(SubmittedVariantInactiveEntity::new) - .collect(Collectors.toList()) + ss1.getAccession(), "Hash mismatch with " + ss1.getClusteredVariantAccession(), + Stream.of(ss1, ss2, ss3, ss4, ss5, ss6).map(SubmittedVariantInactiveEntity::new) + .collect(Collectors.toList()) ); mongoTemplate.insert(Collections.singletonList(splitOperation), SubmittedVariantOperationEntity.class); - rsSplitWriter.write(Collections.singletonList(splitOperation)); + rsSplitWriter.write(Chunk.of(splitOperation)); // Per splitting policy, ensure that SS1 and SS2 get to retain the old RS // Even though locus with start 100, 101 and 102 have equal number of supporting variants @@ -241,7 +216,7 @@ public void testSplitsWithEqualNumberOfDistinctHashes() throws Exception { Collections.singletonList(sameRSAccessionToBeUsedForDifferentLoci)); assertEquals(2, ssAssociatedWithRS1.size()); assertTrue(ssAssociatedWithRS1.stream().map(AccessionWrapper::getAccession).collect(Collectors.toSet()) - .containsAll(Arrays.asList(ss1.getAccession(), ss2.getAccession()))); + .containsAll(Arrays.asList(ss1.getAccession(), ss2.getAccession()))); /* Check clustering counts */ // 2 entries created for hashes with start positions 100 and 102 assertEquals(2, metricCompute.getCount(ClusteringMetric.CLUSTERED_VARIANTS_CREATED)); @@ -256,9 +231,9 @@ public void testSplitsWithEqualNumberOfDistinctHashes() throws Exception { // RS ID created for a given hash in not deterministic during the split. // So, use regex to check for just the hashes in the RS report. assertTrue(rsReportLines.stream() - .anyMatch(line -> line.matches(".*\t717DE940ADEB9E9DCAF09600065BAAF9A91EE713"))); + .anyMatch(line -> line.matches(".*\t717DE940ADEB9E9DCAF09600065BAAF9A91EE713"))); assertTrue(rsReportLines.stream() - .anyMatch(line -> line.matches(".*\t0304A4760D6365E3BAA8E203AF0355248C05979D"))); + .anyMatch(line -> line.matches(".*\t0304A4760D6365E3BAA8E203AF0355248C05979D"))); } @Test @@ -269,23 +244,23 @@ public void testSplitsInvolvingSameSSWithSameRS_Scenario1() throws Exception { ss2 = createSS(2L, rs1Accession, 101L, "T", "A"); // Another SS with the same accession as SS2 but different REF/ALT SubmittedVariantEntity ss2_another = createSS(ss2.getAccession(), rs1Accession, 102L, - "T", ""); + "T", ""); // Create multiple entries in clustered variant collection - one for each distinct loci but with the same accession List multipleEntriesWithSameRS = Stream.of(ss1, ss2, ss2_another) - .map(clusteringWriter::toClusteredVariantEntity) - .collect(Collectors.toList()); + .map(clusteringWriter::toClusteredVariantEntity) + .collect(Collectors.toList()); mongoTemplate.insert(Arrays.asList(ss1, ss2, ss2_another), DbsnpSubmittedVariantEntity.class); mongoTemplate.insert(multipleEntriesWithSameRS, DbsnpClusteredVariantEntity.class); SubmittedVariantOperationEntity splitOperation = new SubmittedVariantOperationEntity(); splitOperation.fill(RSMergeAndSplitCandidatesReaderConfiguration.SPLIT_CANDIDATES_EVENT_TYPE, - ss1.getAccession(), "Hash mismatch with " + ss1.getClusteredVariantAccession(), - Stream.of(ss1, ss2, ss2_another).map(SubmittedVariantInactiveEntity::new) - .collect(Collectors.toList()) + ss1.getAccession(), "Hash mismatch with " + ss1.getClusteredVariantAccession(), + Stream.of(ss1, ss2, ss2_another).map(SubmittedVariantInactiveEntity::new) + .collect(Collectors.toList()) ); mongoTemplate.insert(Collections.singletonList(splitOperation), SubmittedVariantOperationEntity.class); - rsSplitWriter.write(Collections.singletonList(splitOperation)); + rsSplitWriter.write(Chunk.of(splitOperation)); // Even though both SS2 and SS2_another happen to have the same ID and the same RS but different locus // (can happen, see https://www.ebi.ac.uk/panda/jira/browse/EVA-2630), @@ -293,13 +268,13 @@ public void testSplitsInvolvingSameSSWithSameRS_Scenario1() throws Exception { // Here SS1-rsLocus1 gets to keep the old RS ID because SS1 is the older SS ID // SS2 and SS2_another get two different newly issued RS IDs respectively even though they have the same accession Long newRSForSS1 = submittedVariantAccessioningService.get(Collections.singletonList(ss1)).get(0).getData() - .getClusteredVariantAccession(); + .getClusteredVariantAccession(); Long newRSForSS2 = submittedVariantAccessioningService.get(Collections.singletonList(ss2)).get(0).getData() - .getClusteredVariantAccession(); + .getClusteredVariantAccession(); Long newRSForSS2_another = submittedVariantAccessioningService.get(Collections.singletonList(ss2_another)) - .get(0) - .getData() - .getClusteredVariantAccession(); + .get(0) + .getData() + .getClusteredVariantAccession(); assertEquals(rs1Accession, newRSForSS1); assertNotEquals(rs1Accession, newRSForSS2); assertNotEquals(rs1Accession, newRSForSS2_another); @@ -310,9 +285,9 @@ public void testSplitsInvolvingSameSSWithSameRS_Scenario1() throws Exception { // RS ID created for a given hash in not deterministic during the split. // So, use regex to check for just the hashes in the RS report. assertTrue(rsReportLines.stream() - .anyMatch(line -> line.matches(".*\t15F833E1CD27F19EEC66BE89E96B89C99D46A5AB"))); + .anyMatch(line -> line.matches(".*\t15F833E1CD27F19EEC66BE89E96B89C99D46A5AB"))); assertTrue(rsReportLines.stream() - .anyMatch(line -> line.matches(".*\t0304A4760D6365E3BAA8E203AF0355248C05979D"))); + .anyMatch(line -> line.matches(".*\t0304A4760D6365E3BAA8E203AF0355248C05979D"))); } @Test @@ -322,10 +297,10 @@ public void testSplitsInvolvingSameSSWithSameRS_Scenario2() throws Exception { ss1 = createSS(1L, rs1Accession, 100L, "C", "T"); // Another SS with the same accession as SS1 but different REF/ALT SubmittedVariantEntity ss1_another = createSS(ss1.getAccession(), rs1Accession, 101L, - "T", ""); + "T", ""); SubmittedVariantOperationEntity splitOperation = createScenario2_SplitsInvolvingSameSSWithSameRS(ss1, ss1_another, rs1Accession); - rsSplitWriter.write(Collections.singletonList(splitOperation)); + rsSplitWriter.write(Chunk.of(splitOperation)); // Even though the number of supporting variants for rsLocus2 (ss1) and rsLocus3 (ss1_another) are same @@ -336,10 +311,10 @@ public void testSplitsInvolvingSameSSWithSameRS_Scenario2() throws Exception { // RS locus representation of SS1_another - 1_chr1_101_DEL // See ClusteredVariantSplittingPolicy.java Long newRSForSS1 = submittedVariantAccessioningService.get(Collections.singletonList(ss1)).get(0).getData() - .getClusteredVariantAccession(); + .getClusteredVariantAccession(); Long newRSForSS1_another = submittedVariantAccessioningService.get(Collections.singletonList(ss1_another)) - .get(0).getData() - .getClusteredVariantAccession(); + .get(0).getData() + .getClusteredVariantAccession(); assertEquals(rs1Accession, newRSForSS1); assertNotEquals(rs1Accession, newRSForSS1_another); @@ -354,16 +329,16 @@ public void testSplitsInvolvingSameSSWithSameRS_Scenario2() throws Exception { (SubmittedVariantEntity ss1, SubmittedVariantEntity ss1_another, Long rs1Accession) { // Create multiple entries in clustered variant collection - one for each distinct loci but with the same accession List multipleEntriesWithSameRS = Stream.of(ss1, ss1_another) - .map(clusteringWriter::toClusteredVariantEntity) - .collect(Collectors.toList()); + .map(clusteringWriter::toClusteredVariantEntity) + .collect(Collectors.toList()); mongoTemplate.insert(Arrays.asList(ss1, ss1_another), DbsnpSubmittedVariantEntity.class); mongoTemplate.insert(multipleEntriesWithSameRS, DbsnpClusteredVariantEntity.class); SubmittedVariantOperationEntity splitOperation = new SubmittedVariantOperationEntity(); splitOperation.fill(RSMergeAndSplitCandidatesReaderConfiguration.SPLIT_CANDIDATES_EVENT_TYPE, - ss1.getAccession(), "Hash mismatch with " + ss1.getClusteredVariantAccession(), - Stream.of(ss1, ss1_another).map(SubmittedVariantInactiveEntity::new) - .collect(Collectors.toList()) + ss1.getAccession(), "Hash mismatch with " + ss1.getClusteredVariantAccession(), + Stream.of(ss1, ss1_another).map(SubmittedVariantInactiveEntity::new) + .collect(Collectors.toList()) ); mongoTemplate.insert(Collections.singletonList(splitOperation), SubmittedVariantOperationEntity.class); @@ -381,15 +356,15 @@ public void testIdempotentSplitWrites() throws Exception { ss1 = createSS(1L, rs1Accession, 100L, "C", "T"); // Another SS with the same accession as SS1 but different REF/ALT SubmittedVariantEntity ss1_another = createSS(ss1.getAccession(), rs1Accession, 101L, - "T", ""); + "T", ""); SubmittedVariantOperationEntity splitOperation = createScenario2_SplitsInvolvingSameSSWithSameRS(ss1, ss1_another, rs1Accession); - rsSplitWriter.write(Collections.singletonList(splitOperation)); + rsSplitWriter.write(Chunk.of(splitOperation)); DatabaseState databaseStateAfterFirstSplitWrite = DatabaseState.getCurrentDatabaseState(this.mongoTemplate); - rsSplitWriter.write(Collections.singletonList(splitOperation)); + rsSplitWriter.write(Chunk.of(splitOperation)); DatabaseState databaseStateAfterSecondSplitWrite = DatabaseState.getCurrentDatabaseState(this.mongoTemplate); - rsSplitWriter.write(Collections.singletonList(splitOperation)); + rsSplitWriter.write(Chunk.of(splitOperation)); DatabaseState databaseStateAfterThirdSplitWrite = DatabaseState.getCurrentDatabaseState(this.mongoTemplate); assertEquals(databaseStateAfterSecondSplitWrite, databaseStateAfterFirstSplitWrite); @@ -425,7 +400,7 @@ public void testRetainedRSIsNotCreatedWhenHashExists() throws Exception { Stream.of(ss1, ss2).map(SubmittedVariantInactiveEntity::new).collect(Collectors.toList()) ); mongoTemplate.insert(Collections.singletonList(splitOperation), SubmittedVariantOperationEntity.class); - rsSplitWriter.write(Collections.singletonList(splitOperation)); + rsSplitWriter.write(Chunk.of(splitOperation)); // Ensure that SS1 gets to retain the old RS List> ssAssociatedWithRS1 = diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/SSSplitWriterTest.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/SSSplitWriterTest.java index dcba64060..cbe743db0 100644 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/SSSplitWriterTest.java +++ b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/SSSplitWriterTest.java @@ -15,26 +15,22 @@ */ package uk.ac.ebi.eva.accession.clustering.batch.io; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; -import com.mongodb.MongoClient; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mockito; import org.springframework.batch.core.JobExecution; +import org.springframework.batch.item.Chunk; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.test.mock.mockito.MockBean; -import org.springframework.context.ApplicationContext; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.http.HttpMethod; import org.springframework.http.HttpStatus; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import org.springframework.test.web.client.ExpectedCount; import org.springframework.test.web.client.MockRestServiceServer; import org.springframework.web.client.RestTemplate; @@ -44,7 +40,6 @@ import uk.ac.ebi.eva.accession.clustering.test.DatabaseState; import uk.ac.ebi.eva.accession.clustering.test.configuration.BatchTestConfiguration; import uk.ac.ebi.eva.accession.clustering.test.configuration.MongoTestConfiguration; -import uk.ac.ebi.eva.accession.clustering.test.rule.FixSpringMongoDbRule; import uk.ac.ebi.eva.accession.core.model.ISubmittedVariant; import uk.ac.ebi.eva.accession.core.model.SubmittedVariant; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpSubmittedVariantEntity; @@ -53,6 +48,7 @@ import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantOperationEntity; import uk.ac.ebi.eva.accession.core.service.nonhuman.SubmittedVariantAccessioningService; import uk.ac.ebi.eva.accession.core.summary.SubmittedVariantSummaryFunction; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.metrics.count.CountServiceParameters; import uk.ac.ebi.eva.metrics.metric.MetricCompute; @@ -64,9 +60,9 @@ import java.util.stream.Collectors; import java.util.stream.Stream; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.doCallRealMethod; import static org.mockito.Mockito.doThrow; import static org.springframework.test.web.client.match.MockRestRequestMatchers.method; @@ -74,19 +70,14 @@ import static org.springframework.test.web.client.response.MockRestResponseCreators.withStatus; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.SS_SPLIT_WRITER; -@RunWith(SpringRunner.class) +@ExtendWith(SpringExtension.class) @TestPropertySource("classpath:merge-split-test.properties") @ContextConfiguration(classes = {SSSplitWriterConfiguration.class, MongoTestConfiguration.class, BatchTestConfiguration.class}) -public class SSSplitWriterTest { - - private static final String TEST_DB = "test-db"; +public class SSSplitWriterTest extends MongoTestContainerHelper { private static final String ASSEMBLY = "GCA_000000001.1"; - @Autowired - private MongoClient mongoClient; - @Autowired private MongoTemplate mongoTemplate; @@ -94,14 +85,6 @@ public class SSSplitWriterTest { @Qualifier(SS_SPLIT_WRITER) private SSSplitWriter ssSplitWriter; - //Required by nosql-unit - @Autowired - private ApplicationContext applicationContext; - - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName(TEST_DB).build()); - private SubmittedVariantEntity ss1, ss2, ss3, ss4, ss5, ss6; private List svesThatShareSameID; @@ -126,26 +109,24 @@ public class SSSplitWriterTest { private final String URL_PATH_SAVE_COUNT = "/v1/bulk/count"; - @Before + @BeforeEach public void setUp() throws Exception { - cleanup(); + mongoTemplate.getDb().drop(); + metricCompute.clearCount(); + mockServer = MockRestServiceServer.createServer(restTemplate); mockServer.expect(ExpectedCount.manyTimes(), - requestTo(new URI(countServiceParameters.getUrl() + URL_PATH_SAVE_COUNT))) - .andExpect(method(HttpMethod.POST)) - .andRespond(withStatus(HttpStatus.OK)); + requestTo(new URI(countServiceParameters.getUrl() + URL_PATH_SAVE_COUNT))) + .andExpect(method(HttpMethod.POST)) + .andRespond(withStatus(HttpStatus.OK)); Mockito.when(jobExecution.getJobId()).thenReturn(1L); ssSplitWriter.setJobExecution(jobExecution); } - @After + @AfterEach public void tearDown() { - cleanup(); - } - - private void cleanup() { - mongoClient.dropDatabase(TEST_DB); + mongoTemplate.getDb().drop(); metricCompute.clearCount(); } @@ -154,7 +135,7 @@ private SubmittedVariantEntity createSS(Long ssAccession, Long rsAccession, Long Function hashingFunction = new SubmittedVariantSummaryFunction().andThen( new SHA1HashingFunction()); SubmittedVariant temp = new SubmittedVariant(ASSEMBLY, 60711, "PRJ1", "chr1", start, reference, alternate, - rsAccession); + rsAccession); temp.setSupportedByEvidence(supportedByEvidence); temp.setAllelesMatch(allelesMatch); return new SubmittedVariantEntity(ssAccession, hashingFunction.apply(temp), temp, 1); @@ -177,7 +158,7 @@ private void setupSplitScenario() { private void writeSplitWithoutCrashes() throws Exception { setupSplitScenario(); - ssSplitWriter.write(this.svesThatShareSameID); + ssSplitWriter.write(new Chunk<>(this.svesThatShareSameID)); } @Test @@ -198,22 +179,22 @@ private void assertPostSplitStatus() { DbsnpSubmittedVariantEntity.class); assertEquals(3, entriesInDbsnpSVE.size()); List expectedHashesInDbsnpSVE = Stream.of(ss4, ss5, ss6).map(SubmittedVariantEntity::getId) - .collect(Collectors.toList()); + .collect(Collectors.toList()); assertTrue(entriesInDbsnpSVE.stream().map(DbsnpSubmittedVariantEntity::getId) - .allMatch(expectedHashesInDbsnpSVE::contains)); + .allMatch(expectedHashesInDbsnpSVE::contains)); // other SS besides ss4 should be issued new IDs List entriesInSVE = this.mongoTemplate.findAll(SubmittedVariantEntity.class); assertEquals(3, entriesInSVE.size()); assertNotEquals(ss1.getAccession(), - this.submittedVariantAccessioningService.get(Collections.singletonList(ss1)).get(0) - .getAccession()); + this.submittedVariantAccessioningService.get(Collections.singletonList(ss1)).get(0) + .getAccession()); assertNotEquals(ss2.getAccession(), - this.submittedVariantAccessioningService.get(Collections.singletonList(ss2)).get(0) - .getAccession()); + this.submittedVariantAccessioningService.get(Collections.singletonList(ss2)).get(0) + .getAccession()); assertNotEquals(ss3.getAccession(), - this.submittedVariantAccessioningService.get(Collections.singletonList(ss3)).get(0) - .getAccession()); + this.submittedVariantAccessioningService.get(Collections.singletonList(ss3)).get(0) + .getAccession()); // check for creation of 3 new SS IDs assertEquals(3, this.metricCompute.getCount(ClusteringMetric.SUBMITTED_VARIANTS_SS_SPLIT)); @@ -232,14 +213,13 @@ public void testSplitWriterRestartAfterSplitCandidateRegistrationCrashes() throw doThrow(RuntimeException.class).when(mockSplitWriter).registerSplitCandidates(this.svesThatShareSameID); // invoke process - will crash when registering split candidates try { - mockSplitWriter.write(this.svesThatShareSameID); - } - catch (RuntimeException ignored) { + mockSplitWriter.write(new Chunk<>(this.svesThatShareSameID)); + } catch (RuntimeException ignored) { } doCallRealMethod().when(mockSplitWriter).registerSplitCandidates(svesThatShareSameID); // Restart process - mockSplitWriter.write(this.svesThatShareSameID); + mockSplitWriter.write(new Chunk<>(this.svesThatShareSameID)); assertPostSplitStatus(); } @@ -250,14 +230,13 @@ public void testSplitWriterRestartAfterSplitCandidateProcessingCrashes() throws doThrow(RuntimeException.class).when(mockSplitWriter).processSplitCandidates(Mockito.anyList()); // invoke process - will crash when processing split candidates try { - mockSplitWriter.write(this.svesThatShareSameID); - } - catch (RuntimeException ignored) { + mockSplitWriter.write(new Chunk<>(this.svesThatShareSameID)); + } catch (RuntimeException ignored) { } doCallRealMethod().when(mockSplitWriter).processSplitCandidates(Mockito.anyList()); // Restart process - mockSplitWriter.write(this.svesThatShareSameID); + mockSplitWriter.write(new Chunk<>(this.svesThatShareSameID)); assertPostSplitStatus(); } @@ -266,17 +245,16 @@ public void testSplitWriterRestartAfterSplitCandidateClearingCrashes() throws Ex setupSplitScenario(); SSSplitWriter mockSplitWriter = Mockito.spy((SSSplitWriter) ssSplitWriter); doThrow(RuntimeException.class).when(mockSplitWriter) - .removeCurrentSSEntriesInDBForSplitCandidates(Mockito.anySet()); + .removeCurrentSSEntriesInDBForSplitCandidates(Mockito.anySet()); // invoke process - will crash when clearing split candidates try { - mockSplitWriter.write(this.svesThatShareSameID); - } - catch (RuntimeException ignored) { + mockSplitWriter.write(new Chunk<>(this.svesThatShareSameID)); + } catch (RuntimeException ignored) { } doCallRealMethod().when(mockSplitWriter).removeCurrentSSEntriesInDBForSplitCandidates(Mockito.anySet()); // Restart process - mockSplitWriter.write(this.svesThatShareSameID); + mockSplitWriter.write(new Chunk<>(this.svesThatShareSameID)); assertPostSplitStatus(); } @@ -285,7 +263,7 @@ public void testIdempotentSplitWriter() throws Exception { writeSplitWithoutCrashes(); DatabaseState dbStateAfterFirstSplitWrite = DatabaseState.getCurrentDatabaseState(this.mongoTemplate); - ssSplitWriter.write(this.svesThatShareSameID); + ssSplitWriter.write(new Chunk<>(this.svesThatShareSameID)); DatabaseState dbStateAfterSecondSplitWrite = DatabaseState.getCurrentDatabaseState(this.mongoTemplate); assertEquals(dbStateAfterSecondSplitWrite, dbStateAfterFirstSplitWrite); } diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/StudyClusteringMongoReaderTest.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/StudyClusteringMongoReaderTest.java index bbbde7bd6..74c30f7f2 100644 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/StudyClusteringMongoReaderTest.java +++ b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/StudyClusteringMongoReaderTest.java @@ -15,27 +15,22 @@ */ package uk.ac.ebi.eva.accession.clustering.batch.io; -import com.lordofthejars.nosqlunit.annotation.UsingDataSet; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; -import com.mongodb.MongoClient; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.batch.item.ExecutionContext; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.ApplicationContext; +import org.springframework.core.io.ResourceLoader; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; - +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.eva.accession.clustering.test.configuration.MongoTestConfiguration; -import uk.ac.ebi.eva.accession.clustering.test.rule.FixSpringMongoDbRule; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.MongoConfiguration; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; +import uk.ac.ebi.eva.accession.core.utils.MongoTestDataLoader; import java.util.ArrayList; import java.util.Arrays; @@ -44,17 +39,12 @@ import java.util.stream.Collectors; import java.util.stream.Stream; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; -@RunWith(SpringRunner.class) +@ExtendWith(SpringExtension.class) @TestPropertySource("classpath:clustering-pipeline-test.properties") -@UsingDataSet(locations = {"/test-data/submittedVariantEntityStudyReader.json", - "/test-data/dbsnpSubmittedVariantEntityMongoReader.json"}) @ContextConfiguration(classes = {MongoConfiguration.class, MongoTestConfiguration.class}) -public class StudyClusteringMongoReaderTest { - - private static final String TEST_DB = "test-db"; - +public class StudyClusteringMongoReaderTest extends MongoTestContainerHelper { private static final String ASSEMBLY = "GCA_000000001.1"; private static final List PROJECTS = Arrays.asList("projectId_2", "projectId_3"); @@ -69,31 +59,29 @@ public class StudyClusteringMongoReaderTest { private StudyClusteringMongoReader studyClusteringMongoReader; - @Autowired - private MongoClient mongoClient; - @Autowired private MongoTemplate mongoTemplate; - //Required by nosql-unit @Autowired - private ApplicationContext applicationContext; + private ResourceLoader resourceLoader; + + @BeforeEach + public void setUp() { + mongoTemplate.getDb().drop(); - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName(TEST_DB).build()); + MongoTestDataLoader mongoTestDataLoader = new MongoTestDataLoader(mongoTemplate, resourceLoader); + mongoTestDataLoader.load("/test-data/submittedVariantEntityStudyReader.json"); + mongoTestDataLoader.load("/test-data/dbsnpSubmittedVariantEntityMongoReader.json"); - @Before - public void setUp(){ ExecutionContext executionContext = new ExecutionContext(); studyClusteringMongoReader = new StudyClusteringMongoReader(mongoTemplate, ASSEMBLY, PROJECTS, CHUNK_SIZE); studyClusteringMongoReader.open(executionContext); } - @After + @AfterEach public void tearDown() { studyClusteringMongoReader.close(); - mongoClient.dropDatabase(TEST_DB); + mongoTemplate.getDb().drop(); } @Test diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/clustering_writer/ClusteringAssertions.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/clustering_writer/ClusteringAssertions.java index 225d1bdb0..0d052cd98 100644 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/clustering_writer/ClusteringAssertions.java +++ b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/clustering_writer/ClusteringAssertions.java @@ -19,7 +19,7 @@ import uk.ac.ebi.eva.accession.clustering.metric.ClusteringMetric; import uk.ac.ebi.eva.metrics.metric.MetricCompute; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; public class ClusteringAssertions { diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/clustering_writer/IssueAccessionClusteringWriterTest.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/clustering_writer/IssueAccessionClusteringWriterTest.java index 14a8020cc..39f560d3b 100644 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/clustering_writer/IssueAccessionClusteringWriterTest.java +++ b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/clustering_writer/IssueAccessionClusteringWriterTest.java @@ -15,41 +15,37 @@ */ package uk.ac.ebi.eva.accession.clustering.batch.io.clustering_writer; -import com.lordofthejars.nosqlunit.annotation.UsingDataSet; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; import com.mongodb.client.FindIterable; import com.mongodb.client.MongoCollection; import org.bson.Document; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mockito; import org.springframework.batch.core.JobExecution; +import org.springframework.batch.item.Chunk; import org.springframework.batch.item.ExecutionContext; import org.springframework.batch.item.ItemWriter; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.boot.test.mock.mockito.MockBean; -import org.springframework.context.ApplicationContext; +import org.springframework.core.io.ResourceLoader; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.data.mongodb.core.query.Criteria; import org.springframework.data.mongodb.core.query.Query; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.ampt2d.commons.accession.hashing.SHA1HashingFunction; - import uk.ac.ebi.eva.accession.clustering.batch.io.ClusteringMongoReader; import uk.ac.ebi.eva.accession.clustering.batch.io.ClusteringWriter; import uk.ac.ebi.eva.accession.clustering.batch.io.RSSplitWriter; import uk.ac.ebi.eva.accession.clustering.parameters.InputParameters; import uk.ac.ebi.eva.accession.clustering.test.configuration.BatchTestConfiguration; -import uk.ac.ebi.eva.accession.clustering.test.rule.FixSpringMongoDbRule; +import uk.ac.ebi.eva.accession.clustering.test.configuration.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.ClusteredVariantAccessioningConfiguration; import uk.ac.ebi.eva.accession.core.model.ClusteredVariant; import uk.ac.ebi.eva.accession.core.model.IClusteredVariant; @@ -61,6 +57,8 @@ import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantOperationEntity; import uk.ac.ebi.eva.accession.core.summary.ClusteredVariantSummaryFunction; import uk.ac.ebi.eva.accession.core.summary.SubmittedVariantSummaryFunction; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; +import uk.ac.ebi.eva.accession.core.utils.MongoTestDataLoader; import uk.ac.ebi.eva.commons.core.models.VariantType; import uk.ac.ebi.eva.commons.mongodb.readers.MongoDbCursorItemReader; import uk.ac.ebi.eva.metrics.metric.MetricCompute; @@ -74,9 +72,9 @@ import java.util.List; import java.util.function.Function; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import static uk.ac.ebi.eva.accession.clustering.batch.io.clustering_writer.ClusteringAssertions.assertClusteringCounts; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.CLEAR_RS_MERGE_AND_SPLIT_CANDIDATES; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.CLUSTERED_CLUSTERING_WRITER; @@ -88,18 +86,16 @@ /** * This class handles the simplest scenarios of ClusteringWriter. - * + *

* The scenarios tested here are those about issuing new clustered variant accessions (RSs). No reusing existing RSs, * no RSs provided with the submitted variant being clustered, no RS merging. Other test classes in this folder take * care of those scenarios. */ -@RunWith(SpringRunner.class) +@ExtendWith(SpringExtension.class) @EnableAutoConfiguration -@ContextConfiguration(classes = {ClusteredVariantAccessioningConfiguration.class, BatchTestConfiguration.class}) +@ContextConfiguration(classes = {ClusteredVariantAccessioningConfiguration.class, BatchTestConfiguration.class, MongoTestConfiguration.class}) @TestPropertySource("classpath:clustering-issuance-test.properties") -public class IssueAccessionClusteringWriterTest { - - private static final String TEST_DB = "test-db"; +public class IssueAccessionClusteringWriterTest extends MongoTestContainerHelper { private static final String CLUSTERED_VARIANT_COLLECTION = "clusteredVariantEntity"; @@ -119,11 +115,10 @@ public class IssueAccessionClusteringWriterTest { private MongoTemplate mongoTemplate; @Autowired - private MetricCompute metricCompute; + private ResourceLoader resourceLoader; - //Required by nosql-unit @Autowired - private ApplicationContext applicationContext; + private MetricCompute metricCompute; // Current clustering sequence is: // generate merge split candidates from clustered variants -> perform merge @@ -163,12 +158,9 @@ public class IssueAccessionClusteringWriterTest { private Function clusteredHashingFunction; - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName(TEST_DB).build()); - - @Before + @BeforeEach public void setUp() throws IOException { + mongoTemplate.getDb().drop(); hashingFunction = new SubmittedVariantSummaryFunction().andThen(new SHA1HashingFunction()); clusteredHashingFunction = new ClusteredVariantSummaryFunction().andThen(new SHA1HashingFunction()); Files.deleteIfExists(this.rsReportFile.toPath()); @@ -179,16 +171,17 @@ public void setUp() throws IOException { clusteringWriterPreMergeAndSplit.setJobExecution(jobExecution); } - @After + @AfterEach public void tearDown() { mongoTemplate.getDb().drop(); metricCompute.clearCount(); } @Test - @UsingDataSet(locations = {"/test-data/submittedVariantEntity.json"}) @DirtiesContext public void writer() throws Exception { + new MongoTestDataLoader(mongoTemplate, resourceLoader).load("/test-data/submittedVariantEntity.json"); + List submittedVariantEntities = createSubmittedVariantEntities(); this.clusterVariants(submittedVariantEntities); assertClusteredVariantsCreated(); @@ -206,22 +199,22 @@ public void writer() throws Exception { private List createSubmittedVariantEntities() { List submittedVariantEntities = new ArrayList<>(); SubmittedVariant submittedVariant1 = createSubmittedVariant(inputParameters.getAssemblyAccession(), 1000, - PROJECT_ACCESSION, "1", 1000L, "T", "A"); + PROJECT_ACCESSION, "1", 1000L, "T", "A"); SubmittedVariantEntity submittedVariantEntity1 = createSubmittedVariantEntity(5000000001L, submittedVariant1); //Different alleles SubmittedVariant submittedVariant2 = createSubmittedVariant(inputParameters.getAssemblyAccession(), 1000, - PROJECT_ACCESSION, "1", 1000L, "T", "G"); + PROJECT_ACCESSION, "1", 1000L, "T", "G"); SubmittedVariantEntity submittedVariantEntity2 = createSubmittedVariantEntity(5000000002L, submittedVariant2); //Same assembly, contig, start but different type SubmittedVariant submittedVariantINS = createSubmittedVariant(inputParameters.getAssemblyAccession(), 1000, - PROJECT_ACCESSION, "1", 1000L, "", "A"); + PROJECT_ACCESSION, "1", 1000L, "", "A"); SubmittedVariantEntity submittedVariantEntityINS = createSubmittedVariantEntity(5000000003L, submittedVariantINS); SubmittedVariant submittedVariantDEL = createSubmittedVariant(inputParameters.getAssemblyAccession(), 1000, - PROJECT_ACCESSION, "1", 1000L, "T", ""); + PROJECT_ACCESSION, "1", 1000L, "T", ""); SubmittedVariantEntity submittedVariantEntityDEL = createSubmittedVariantEntity(5000000004L, submittedVariantDEL); //Different assembly, contig and start SubmittedVariant submittedVariant3 = createSubmittedVariant(inputParameters.getAssemblyAccession(), 3000, - PROJECT_ACCESSION, "1", 3000L, "C", "G"); + PROJECT_ACCESSION, "1", 3000L, "C", "G"); SubmittedVariantEntity submittedVariantEntity3 = createSubmittedVariantEntity(5000000005L, submittedVariant3); submittedVariantEntities.add(submittedVariantEntity1); submittedVariantEntities.add(submittedVariantEntity2); @@ -245,7 +238,7 @@ private SubmittedVariant createSubmittedVariant(String referenceSequenceAccessio String projectAccession, String contig, long start, String referenceAllele, String alternateAllele) { return new SubmittedVariant(referenceSequenceAccession, taxonomyAccession, projectAccession, contig, start, - referenceAllele, alternateAllele, null); + referenceAllele, alternateAllele, null); } private void assertClusteredVariantsCreated() { @@ -278,14 +271,14 @@ private boolean allSubmittedVariantsClustered() { MongoCollection collection = mongoTemplate.getCollection(SUBMITTED_VARIANT_COLLECTION); FindIterable documents = collection.find(); for (Document document : documents) { - if (document.get("rs") == null){ + if (document.get("rs") == null) { return false; } } return true; } - private void assertSubmittedVariantsOperationInserted(){ + private void assertSubmittedVariantsOperationInserted() { MongoCollection collection = mongoTemplate.getCollection(SUBMITTED_VARIANT_OPERATION_COLLECTION); assertEquals(5, collection.countDocuments()); List expectedAccessions = Arrays.asList(5000000001L, 5000000002L, 5000000003L, 5000000004L, 5000000005L); @@ -300,11 +293,11 @@ public void cluster_eva_submitted_variant_into_a_dbsnp_not_multimap_clustered_va // given Long rsAccession = 30L; ClusteredVariantEntity rs1 = createClusteredVariantEntity(rsAccession, inputParameters.getAssemblyAccession(), - "chr1", null); + "chr1", null); SubmittedVariantEntity ssToCluster = createSubmittedVariantEntity(5200000000L, - inputParameters.getAssemblyAccession(), - "chr1", "project2", null); + inputParameters.getAssemblyAccession(), + "chr1", "project2", null); mongoTemplate.insert(Arrays.asList(rs1), DbsnpClusteredVariantEntity.class); mongoTemplate.insert(Arrays.asList(ssToCluster), SubmittedVariantEntity.class); @@ -328,27 +321,27 @@ public void do_not_cluster_eva_submitted_variant_into_a_dbsnp_multimap_clustered throws Exception { // given ClusteredVariantEntity rs1Locus1 = createClusteredVariantEntity(30L, inputParameters.getAssemblyAccession(), - "chr1", 3); + "chr1", 3); ClusteredVariantEntity rs1Locus2 = createClusteredVariantEntity(30L, inputParameters.getAssemblyAccession(), - "chr2", null); + "chr2", null); SubmittedVariantEntity ssToCluster = createSubmittedVariantEntity(5200000000L, - inputParameters.getAssemblyAccession(), - "chr1", "project2", - null); + inputParameters.getAssemblyAccession(), + "chr1", "project2", + null); mongoTemplate.insert(Arrays.asList(rs1Locus1, rs1Locus2), DbsnpClusteredVariantEntity.class); mongoTemplate.insert(Arrays.asList(ssToCluster), SubmittedVariantEntity.class); Query querySsToCluster = new Query(new Criteria("accession").is(ssToCluster.getAccession())); assertNull(mongoTemplate.find(querySsToCluster, SubmittedVariantEntity.class).get(0) - .getClusteredVariantAccession()); + .getClusteredVariantAccession()); // when this.clusterVariants(Collections.singletonList(ssToCluster)); // then assertNull(mongoTemplate.find(querySsToCluster, SubmittedVariantEntity.class).get(0) - .getClusteredVariantAccession()); + .getClusteredVariantAccession()); assertClusteringCounts(metricCompute, 0, 0, 0, 1, 0, 0, 0); } @@ -357,11 +350,11 @@ public void do_not_cluster_eva_submitted_variant_into_a_dbsnp_multimap_clustered public void do_not_cluster_eva_submitted_variant_into_a_dbsnp_multimap_clustered_variant() throws Exception { // given ClusteredVariantEntity rs1 = createClusteredVariantEntity(30L, inputParameters.getAssemblyAccession(), "chr1", - 3); + 3); SubmittedVariantEntity ssToCluster = createSubmittedVariantEntity(5200000000L, - inputParameters.getAssemblyAccession(), - "chr1", "project2", - null); + inputParameters.getAssemblyAccession(), + "chr1", "project2", + null); mongoTemplate.insert(Arrays.asList(rs1), DbsnpClusteredVariantEntity.class); mongoTemplate.insert(Arrays.asList(ssToCluster), SubmittedVariantEntity.class); @@ -388,8 +381,8 @@ private ClusteredVariantEntity createClusteredVariantEntity(Long accession, Stri ClusteredVariant variant = createClusteredVariant(assemblyAccession, contig); String hash = clusteredHashingFunction.apply(variant); ClusteredVariantEntity variantEntity = new ClusteredVariantEntity(accession, hash, assemblyAccession, 1000, - contig, 100, VariantType.SNV, false, null, 1, - mapWeight); + contig, 100, VariantType.SNV, false, null, 1, + mapWeight); return variantEntity; } @@ -405,38 +398,38 @@ private SubmittedVariantEntity createSubmittedVariantEntity(Long accession, Stri private void clusterVariants(List submittedVariantEntities) throws Exception { - clusteringWriterPreMergeAndSplit.write(submittedVariantEntities); + clusteringWriterPreMergeAndSplit.write(new Chunk<>(submittedVariantEntities)); List mergeCandidates = new ArrayList<>(); List splitCandidates = new ArrayList<>(); SubmittedVariantOperationEntity tempSVO; rsMergeCandidatesReader.open(new ExecutionContext()); - while((tempSVO = rsMergeCandidatesReader.read()) != null) { + while ((tempSVO = rsMergeCandidatesReader.read()) != null) { mergeCandidates.add(tempSVO); } rsSplitCandidatesReader.open(new ExecutionContext()); - while((tempSVO = rsSplitCandidatesReader.read()) != null) { + while ((tempSVO = rsSplitCandidatesReader.read()) != null) { splitCandidates.add(tempSVO); } - rsMergeWriter.write(mergeCandidates); - rsSplitWriter.write(splitCandidates); + rsMergeWriter.write(new Chunk<>(mergeCandidates)); + rsSplitWriter.write(new Chunk<>(splitCandidates)); ClusteringMongoReader unclusteredVariantsReader = new ClusteringMongoReader(this.mongoTemplate, - inputParameters.getAssemblyAccession(), - 100, - false); + inputParameters.getAssemblyAccession(), + 100, + false); unclusteredVariantsReader.initializeReader(); List unclusteredVariants = new ArrayList<>(); SubmittedVariantEntity tempSV; - while((tempSV = unclusteredVariantsReader.read()) != null) { + while ((tempSV = unclusteredVariantsReader.read()) != null) { unclusteredVariants.add(tempSV); } unclusteredVariantsReader.close(); // Cluster non-clustered variants - clusteringWriterPostMergeAndSplit.write(unclusteredVariants); + clusteringWriterPostMergeAndSplit.write(new Chunk<>(unclusteredVariants)); // Spring has a mandatory requirement of even small functionality being writers. // To satisfy that, we pass in a dummy object to invoke the writer // which basically clears the merge and split operations after they were processed above - clearRSMergeAndSplitCandidates.write(Collections.singletonList(new Object())); + clearRSMergeAndSplitCandidates.write(Chunk.of(new Object())); } } diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/clustering_writer/MergeAccessionClusteringWriterTest.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/clustering_writer/MergeAccessionClusteringWriterTest.java index b0ef3d4e2..dd014267f 100644 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/clustering_writer/MergeAccessionClusteringWriterTest.java +++ b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/clustering_writer/MergeAccessionClusteringWriterTest.java @@ -15,35 +15,30 @@ */ package uk.ac.ebi.eva.accession.clustering.batch.io.clustering_writer; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; import org.assertj.core.util.Sets; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.batch.item.Chunk; import org.springframework.batch.item.ExecutionContext; import org.springframework.batch.item.ItemWriter; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; -import org.springframework.context.ApplicationContext; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.data.mongodb.core.query.Query; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; - +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.ampt2d.commons.accession.core.models.EventType; import uk.ac.ebi.ampt2d.commons.accession.hashing.SHA1HashingFunction; import uk.ac.ebi.ampt2d.commons.accession.persistence.mongodb.document.EventDocument; - import uk.ac.ebi.eva.accession.clustering.batch.io.ClusteringMongoReader; import uk.ac.ebi.eva.accession.clustering.batch.io.ClusteringWriter; import uk.ac.ebi.eva.accession.clustering.test.configuration.BatchTestConfiguration; -import uk.ac.ebi.eva.accession.clustering.test.rule.FixSpringMongoDbRule; +import uk.ac.ebi.eva.accession.clustering.test.configuration.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.ClusteredVariantAccessioningConfiguration; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.SubmittedVariantAccessioningConfiguration; import uk.ac.ebi.eva.accession.core.model.ClusteredVariant; @@ -62,6 +57,7 @@ import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantOperationEntity; import uk.ac.ebi.eva.accession.core.summary.ClusteredVariantSummaryFunction; import uk.ac.ebi.eva.accession.core.summary.SubmittedVariantSummaryFunction; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.commons.core.models.VariantType; import uk.ac.ebi.eva.commons.mongodb.readers.MongoDbCursorItemReader; import uk.ac.ebi.eva.metrics.metric.MetricCompute; @@ -74,9 +70,9 @@ import java.util.function.Function; import java.util.stream.Collectors; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import static uk.ac.ebi.eva.accession.clustering.batch.io.clustering_writer.ClusteringAssertions.assertClusteringCounts; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.CLEAR_RS_MERGE_AND_SPLIT_CANDIDATES; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.CLUSTERED_CLUSTERING_WRITER; @@ -92,20 +88,17 @@ /** * This class handles some scenarios of ClusteringWriter where redundant RSs are discovered and merged. - * + *

* That includes all the scenarios that involve writing to the Operation collections, e.g. * clusteredVariantOperationEntity to register an RS merge. * Other test classes in this folder take care of other scenarios. */ -@RunWith(SpringRunner.class) +@ExtendWith(SpringExtension.class) @EnableAutoConfiguration @ContextConfiguration(classes = {ClusteredVariantAccessioningConfiguration.class, - SubmittedVariantAccessioningConfiguration.class, BatchTestConfiguration.class}) + SubmittedVariantAccessioningConfiguration.class, BatchTestConfiguration.class, MongoTestConfiguration.class}) @TestPropertySource("classpath:clustering-writer-test.properties") -public class MergeAccessionClusteringWriterTest { - - private static final String TEST_DB = "test-db"; - +public class MergeAccessionClusteringWriterTest extends MongoTestContainerHelper { private static final String CLUSTERED_VARIANT_COLLECTION = "clusteredVariantEntity"; private static final String SUBMITTED_VARIANT_COLLECTION = "submittedVariantEntity"; @@ -166,30 +159,23 @@ public class MergeAccessionClusteringWriterTest { @Autowired private Long accessioningMonotonicInitSs; - //Required by nosql-unit - @Autowired - private ApplicationContext applicationContext; - private Function hashingFunction; private Function clusteredHashingFunction; - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName(TEST_DB).build()); - private static final String defaultReferenceAlleleForTesting = "T"; private static final String defaultAlternateAlleleForTesting = "A"; - @Before + @BeforeEach public void setUp() { mongoTemplate.getDb().drop(); + metricCompute.clearCount(); hashingFunction = new SubmittedVariantSummaryFunction().andThen(new SHA1HashingFunction()); clusteredHashingFunction = new ClusteredVariantSummaryFunction().andThen(new SHA1HashingFunction()); } - @After + @AfterEach public void tearDown() { mongoTemplate.getDb().drop(); metricCompute.clearCount(); @@ -287,20 +273,20 @@ public void mergeClusteredAccession(Long rs1, Long rs2, Long ssToRemap, Long ss2 // ssToRemap in the old assembly, will be remapped to asm2 (see sve1Remapped below) mongoTemplate.insert(createSubmittedVariantEntity(ASM_1, START, rs1, ssToRemap, NOT_REMAPPED), - getSubmittedCollection(ssToRemap)); + getSubmittedCollection(ssToRemap)); // ss2 in the new assembly, will change its RS to rs1 when we realise rs1 and rs2 should be merged // because they both map to the same position in asm2 mongoTemplate.insert(createSubmittedVariantEntity(ASM_2, START, rs2, ss2, NOT_REMAPPED), getSubmittedCollection(ss2)); assertDatabaseCounts(expectedDbsnpCve - additionalDbsnpCveCreatedPostMerge, - expectedCve - additionalCveCreatedPostMerge, 0, 0, - expectedDbsnpSve, expectedSve, 0, 0); + expectedCve - additionalCveCreatedPostMerge, 0, 0, + expectedDbsnpSve, expectedSve, 0, 0); // when SubmittedVariantEntity sve1Remapped = createSubmittedVariantEntity(ASM_2, START, rs1, ssToRemap, ASM_1, - defaultAlternateAlleleForTesting, - defaultReferenceAlleleForTesting); + defaultAlternateAlleleForTesting, + defaultReferenceAlleleForTesting); // To ensure that we don't create collision with ss2, // use alleles other than defaultAlternateAlleleForTesting or defaultReferenceAlleleForTesting mongoTemplate.insert(sve1Remapped, getSubmittedCollection(sve1Remapped.getAccession())); @@ -314,7 +300,7 @@ public void mergeClusteredAccession(Long rs1, Long rs2, Long ssToRemap, Long ss2 // then assertDatabaseCounts(expectedDbsnpCve, expectedCve, expectedDbsnpCvOperations, expectedCvOperations, - expectedDbsnpSve, expectedSve, expectedDbsnpSvOperations, expectedSvOperations); + expectedDbsnpSve, expectedSve, expectedDbsnpSvOperations, expectedSvOperations); assertAssembliesPresent(Sets.newTreeSet(ASM_1, ASM_2)); } @@ -325,12 +311,11 @@ public void mergeClusteredAccession(Long rs1, Long rs2, Long ssToRemap, Long ss2 int expectedDbsnpSve, int expectedSve, int expectedDbsnpSvOperations, int expectedSvOperations) throws Exception { mergeClusteredAccession(rs1, rs2, ssToRemap, ss2, expectedDbsnpCve, expectedCve, expectedDbsnpCvOperations, - expectedCvOperations, expectedDbsnpSve, expectedSve, expectedDbsnpSvOperations, - expectedSvOperations, 0, 0); + expectedCvOperations, expectedDbsnpSve, expectedSve, expectedDbsnpSvOperations, + expectedSvOperations, 0, 0); } - private ClusteredVariantEntity createClusteredVariantEntity(String assembly, Long rs) { ClusteredVariant cv = new ClusteredVariant(assembly, 1000, "1", 100L, VariantType.SNV, false, null); String cvHash = clusteredHashingFunction.apply(cv); @@ -360,12 +345,12 @@ private void assertDatabaseCounts(int expectedDbsnpCve, int expectedCve, assertEquals(expectedDbsnpCve, mongoTemplate.count(new Query(), DbsnpClusteredVariantEntity.class)); assertEquals(expectedCve, mongoTemplate.count(new Query(), ClusteredVariantEntity.class)); assertEquals(expectedDbsnpCvOperations, mongoTemplate.count(new Query(), - DbsnpClusteredVariantOperationEntity.class)); + DbsnpClusteredVariantOperationEntity.class)); assertEquals(expectedCvOperations, mongoTemplate.count(new Query(), ClusteredVariantOperationEntity.class)); assertEquals(expectedDbsnpSve, mongoTemplate.count(new Query(), DbsnpSubmittedVariantEntity.class)); assertEquals(expectedSve, mongoTemplate.count(new Query(), SubmittedVariantEntity.class)); assertEquals(expectedDbsnpSvOperations, mongoTemplate.count(new Query(), - DbsnpSubmittedVariantOperationEntity.class)); + DbsnpSubmittedVariantOperationEntity.class)); assertEquals(expectedSvOperations, mongoTemplate.count(new Query(), SubmittedVariantOperationEntity.class)); } @@ -373,14 +358,14 @@ private void assertMergedInto(Long mergedInto, Long originalAccession, Long... u // Confine the variants being inspected to the remapped assembly List submittedVariants = mongoTemplate - .findAll(SubmittedVariantEntity.class) - .stream().filter(entity -> entity.getReferenceSequenceAccession().equals(ASM_2)) - .collect(Collectors.toList()); + .findAll(SubmittedVariantEntity.class) + .stream().filter(entity -> entity.getReferenceSequenceAccession().equals(ASM_2)) + .collect(Collectors.toList()); submittedVariants.addAll(mongoTemplate - .findAll(DbsnpSubmittedVariantEntity.class) - .stream() - .filter(entity -> entity.getReferenceSequenceAccession().equals(ASM_2)) - .collect(Collectors.toList())); + .findAll(DbsnpSubmittedVariantEntity.class) + .stream() + .filter(entity -> entity.getReferenceSequenceAccession().equals(ASM_2)) + .collect(Collectors.toList())); assertClusteredVariantAccessionEqual(Sets.newTreeSet(mergedInto), submittedVariants); // Confine the variants being inspected to the remapped assembly @@ -390,9 +375,9 @@ private void assertMergedInto(Long mergedInto, Long originalAccession, Long... u .stream().filter(entity -> entity.getAssemblyAccession().equals(ASM_2)) .collect(Collectors.toList()); clusteredVariants.addAll(mongoTemplate - .findAll(DbsnpClusteredVariantEntity.class) - .stream().filter(entity -> entity.getAssemblyAccession().equals(ASM_2)) - .collect(Collectors.toList())); + .findAll(DbsnpClusteredVariantEntity.class) + .stream().filter(entity -> entity.getAssemblyAccession().equals(ASM_2)) + .collect(Collectors.toList())); assertAccessionEqual(Sets.newTreeSet(mergedInto), clusteredVariants); EventDocument clusteredOp = @@ -412,8 +397,8 @@ private void assertMergedInto(Long mergedInto, Long originalAccession, Long... u } assertTrue(submittedOps.stream().map(EventDocument::getEventType).allMatch(EventType.UPDATED::equals)); assertEquals(Sets.newTreeSet(updatedSubmittedVariant), submittedOps.stream() - .map(EventDocument::getAccession) - .collect(Collectors.toSet())); + .map(EventDocument::getAccession) + .collect(Collectors.toSet())); } @Test @@ -433,7 +418,7 @@ public void do_not_merge_into_remapped_multimap_variants() throws Exception { mongoTemplate.insert(createClusteredVariantEntity(asm1, 200L, rs1, null), getClusteredTable(rs1)); mongoTemplate.insert(createSubmittedVariantEntity(asm1, 100L, rs1, ssToRemap, NOT_REMAPPED), - getSubmittedCollection(ssToRemap)); + getSubmittedCollection(ssToRemap)); mongoTemplate.insert(createSubmittedVariantEntity(asm1, 200L, rs1, ss3, NOT_REMAPPED), getSubmittedCollection(ssToRemap)); @@ -443,7 +428,7 @@ public void do_not_merge_into_remapped_multimap_variants() throws Exception { mongoTemplate.insert(createSubmittedVariantEntity(asm2, 100L, rs2, ss2, NOT_REMAPPED), getSubmittedCollection(ss2)); assertDatabaseCounts(0, 3, 0, 0, - 0, 3, 0, 0); + 0, 3, 0, 0); // when SubmittedVariantEntity sve1Remapped = createSubmittedVariantEntity(asm2, 100L, rs1, ssToRemap, asm1); @@ -452,7 +437,7 @@ public void do_not_merge_into_remapped_multimap_variants() throws Exception { // then assertDatabaseCounts(0, 3, 0, 0, - 0, 3, 0, 0); + 0, 3, 0, 0); assertAssembliesPresent(Sets.newTreeSet(asm1, asm2)); } @@ -475,7 +460,7 @@ public void do_not_merge_remapped_multimap_variants() throws Exception { mongoTemplate.insert(createClusteredVariantEntity(asm1, 200L, rs1, 3), getClusteredTable(rs1)); mongoTemplate.insert(createSubmittedVariantEntity(asm1, 100L, rs1, ssToRemap, NOT_REMAPPED), - getSubmittedCollection(ssToRemap)); + getSubmittedCollection(ssToRemap)); mongoTemplate.insert(createSubmittedVariantEntity(asm1, 200L, rs1, ss3, NOT_REMAPPED), getSubmittedCollection(ssToRemap)); @@ -484,7 +469,7 @@ public void do_not_merge_remapped_multimap_variants() throws Exception { mongoTemplate.insert(createSubmittedVariantEntity(asm2, 100L, rs2, ss2, NOT_REMAPPED), getSubmittedCollection(ss2)); assertDatabaseCounts(0, 3, 0, 0, - 0, 3, 0, 0); + 0, 3, 0, 0); // when SubmittedVariantEntity sve1Remapped = createSubmittedVariantEntity(asm2, 100L, rs1, ssToRemap, asm1); @@ -493,7 +478,7 @@ public void do_not_merge_remapped_multimap_variants() throws Exception { // then assertDatabaseCounts(0, 3, 0, 0, - 0, 3, 0, 0); + 0, 3, 0, 0); assertAssembliesPresent(Sets.newTreeSet(asm1, asm2)); } @@ -526,7 +511,7 @@ public void merge_into_remapped_multimap_variants_if_single_mapping_per_assembly mongoTemplate.insert(createSubmittedVariantEntity(asm2, 200L, rs2, ss2, NOT_REMAPPED), getSubmittedCollection(ss2)); assertDatabaseCounts(0, 3, 0, 0, - 0, 3, 0, 0); + 0, 3, 0, 0); // when SubmittedVariantEntity sve1Remapped = createSubmittedVariantEntity(asm2, 200L, rs1, ssToRemap, asm1); @@ -534,7 +519,7 @@ public void merge_into_remapped_multimap_variants_if_single_mapping_per_assembly // then assertDatabaseCounts(0, 3, 0, 1, - 0, 3, 0, 1); + 0, 3, 0, 1); assertAssembliesPresent(Sets.newTreeSet(asm1, asm2, asm3)); assertMergedInto(rs1, rs2, ss2); @@ -568,7 +553,7 @@ public void merge_multimap_variants_into_remapped_variants_if_single_mapping_per mongoTemplate.insert(createSubmittedVariantEntity(asm3, 300L, rs2, ss3, NOT_REMAPPED), getSubmittedCollection(ss3)); assertDatabaseCounts(0, 3, 0, 0, - 0, 3, 0, 0); + 0, 3, 0, 0); // when SubmittedVariantEntity sve1Remapped = createSubmittedVariantEntity(asm2, 200L, rs1, ssToRemap, asm1); @@ -577,7 +562,7 @@ public void merge_multimap_variants_into_remapped_variants_if_single_mapping_per // then // RS2 to RS1 merge will be confined to the ASM2 assembly. Therefore number of CV and SV operations is just 1. assertDatabaseCounts(0, 3, 0, 1, - 0, 3, 0, 1); + 0, 3, 0, 1); assertAssembliesPresent(Sets.newTreeSet(asm1, asm2, asm3)); assertMergedInto(rs1, rs2, ss2); @@ -598,7 +583,7 @@ private ClusteredVariantEntity createClusteredVariantEntity(String assembly, Lon ClusteredVariant cv = new ClusteredVariant(assembly, 1000, "1", start, VariantType.SNV, false, null); String cvHash = clusteredHashingFunction.apply(cv); return new ClusteredVariantEntity(rs, cvHash, assembly, 1000, "1", start, VariantType.SNV, false, null, 1, - mapWeight); + mapWeight); } private SubmittedVariantEntity createSubmittedVariantEntity(String assembly, Long start, Long rs, Long ss, @@ -607,8 +592,8 @@ private SubmittedVariantEntity createSubmittedVariantEntity(String assembly, Lon SubmittedVariant submittedClustered = new SubmittedVariant(assembly, 1000, "project", "1", start, reference, alternate, rs); String hash1 = hashingFunction.apply(submittedClustered); SubmittedVariantEntity submittedVariantEntity = new SubmittedVariantEntity(ss, hash1, submittedClustered, 1, - remappedFrom, LocalDateTime.now(), - null); + remappedFrom, LocalDateTime.now(), + null); submittedVariantEntity.setMapWeight(mapWeight); return submittedVariantEntity; } @@ -616,53 +601,53 @@ private SubmittedVariantEntity createSubmittedVariantEntity(String assembly, Lon private SubmittedVariantEntity createSubmittedVariantEntity(String assembly, Long start, Long rs, Long ss, String remappedFrom, Integer mapWeight) { return createSubmittedVariantEntity(assembly, start, rs, ss, remappedFrom, mapWeight, - defaultReferenceAlleleForTesting, defaultAlternateAlleleForTesting); + defaultReferenceAlleleForTesting, defaultAlternateAlleleForTesting); } private SubmittedVariantEntity createSubmittedVariantEntity(String assembly, Long start, Long rs, Long ss, String remappedFrom) { return createSubmittedVariantEntity(assembly, start, rs, ss, remappedFrom, null, - defaultReferenceAlleleForTesting, defaultAlternateAlleleForTesting); + defaultReferenceAlleleForTesting, defaultAlternateAlleleForTesting); } private SubmittedVariantEntity createSubmittedVariantEntity(String assembly, Long start, Long rs, Long ss, String remappedFrom, String reference, String alternate) { return createSubmittedVariantEntity(assembly, start, rs, ss, remappedFrom, null, - reference, alternate); + reference, alternate); } private void clusterVariants(List submittedVariantEntities) throws Exception { - clusteringWriterPreMergeAndSplit.write(submittedVariantEntities); + clusteringWriterPreMergeAndSplit.write(new Chunk<>(submittedVariantEntities)); List mergeCandidates = new ArrayList<>(); List splitCandidates = new ArrayList<>(); SubmittedVariantOperationEntity tempSVO; rsMergeCandidatesReader.open(new ExecutionContext()); - while((tempSVO = rsMergeCandidatesReader.read()) != null) { + while ((tempSVO = rsMergeCandidatesReader.read()) != null) { mergeCandidates.add(tempSVO); } rsSplitCandidatesReader.open(new ExecutionContext()); - while((tempSVO = rsSplitCandidatesReader.read()) != null) { + while ((tempSVO = rsSplitCandidatesReader.read()) != null) { splitCandidates.add(tempSVO); } - rsMergeWriter.write(mergeCandidates); - rsSplitWriter.write(splitCandidates); + rsMergeWriter.write(new Chunk<>(mergeCandidates)); + rsSplitWriter.write(new Chunk<>(splitCandidates)); ClusteringMongoReader unclusteredVariantsReader = new ClusteringMongoReader(this.mongoTemplate, ASM_2, 100, - false); + false); unclusteredVariantsReader.initializeReader(); List unclusteredVariants = new ArrayList<>(); SubmittedVariantEntity tempSV; - while((tempSV = unclusteredVariantsReader.read()) != null) { + while ((tempSV = unclusteredVariantsReader.read()) != null) { unclusteredVariants.add(tempSV); } unclusteredVariantsReader.close(); // Cluster non-clustered variants - clusteringWriterPostMergeAndSplit.write(unclusteredVariants); + clusteringWriterPostMergeAndSplit.write(new Chunk<>(unclusteredVariants)); // Spring has a mandatory requirement of even small functionality being writers. // To satisfy that, we pass in a dummy object to invoke the writer // which basically clears the merge and split operations after they were processed above - clearRSMergeAndSplitCandidates.write(Collections.singletonList(new Object())); + clearRSMergeAndSplitCandidates.write(Chunk.of(new Object())); } } diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/clustering_writer/RemappedVariantsClusteringWriterTest.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/clustering_writer/RemappedVariantsClusteringWriterTest.java index 05f1b2f27..95be884ea 100644 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/clustering_writer/RemappedVariantsClusteringWriterTest.java +++ b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/clustering_writer/RemappedVariantsClusteringWriterTest.java @@ -15,36 +15,32 @@ */ package uk.ac.ebi.eva.accession.clustering.batch.io.clustering_writer; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mockito; import org.springframework.batch.core.JobExecution; +import org.springframework.batch.item.Chunk; import org.springframework.batch.item.ExecutionContext; import org.springframework.batch.item.ItemWriter; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.boot.test.mock.mockito.MockBean; -import org.springframework.context.ApplicationContext; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.ampt2d.commons.accession.core.models.EventType; import uk.ac.ebi.ampt2d.commons.accession.hashing.SHA1HashingFunction; - import uk.ac.ebi.eva.accession.clustering.batch.io.ClusteringMongoReader; import uk.ac.ebi.eva.accession.clustering.batch.io.ClusteringWriter; import uk.ac.ebi.eva.accession.clustering.batch.io.RSSplitWriter; import uk.ac.ebi.eva.accession.clustering.parameters.InputParameters; import uk.ac.ebi.eva.accession.clustering.test.configuration.BatchTestConfiguration; -import uk.ac.ebi.eva.accession.clustering.test.rule.FixSpringMongoDbRule; +import uk.ac.ebi.eva.accession.clustering.test.configuration.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.ClusteredVariantAccessioningConfiguration; import uk.ac.ebi.eva.accession.core.model.ClusteredVariant; import uk.ac.ebi.eva.accession.core.model.IClusteredVariant; @@ -57,6 +53,7 @@ import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantOperationEntity; import uk.ac.ebi.eva.accession.core.summary.ClusteredVariantSummaryFunction; import uk.ac.ebi.eva.accession.core.summary.SubmittedVariantSummaryFunction; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.commons.core.models.VariantType; import uk.ac.ebi.eva.commons.mongodb.readers.MongoDbCursorItemReader; @@ -70,8 +67,8 @@ import java.util.function.Function; import java.util.stream.Collectors; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.BACK_PROPAGATED_RS_WRITER; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.CLEAR_RS_MERGE_AND_SPLIT_CANDIDATES; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.CLUSTERED_CLUSTERING_WRITER; @@ -86,14 +83,12 @@ *

* Other test classes in this folder take care of other scenarios. */ -@RunWith(SpringRunner.class) +@ExtendWith(SpringExtension.class) @EnableAutoConfiguration -@ContextConfiguration(classes = {ClusteredVariantAccessioningConfiguration.class, BatchTestConfiguration.class}) +@ContextConfiguration(classes = {ClusteredVariantAccessioningConfiguration.class, BatchTestConfiguration.class, + MongoTestConfiguration.class}) @TestPropertySource("classpath:clustering-writer-test.properties") -public class RemappedVariantsClusteringWriterTest { - - private static final String TEST_DB = "test-db"; - +public class RemappedVariantsClusteringWriterTest extends MongoTestContainerHelper { private static final String CLUSTERED_VARIANT_COLLECTION = "clusteredVariantEntity"; private static final String SUBMITTED_VARIANT_COLLECTION = "submittedVariantEntity"; @@ -160,15 +155,7 @@ public class RemappedVariantsClusteringWriterTest { private Function clusteredHashingFunction; - //Required by nosql-unit - @Autowired - private ApplicationContext applicationContext; - - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName(TEST_DB).build()); - - @Before + @BeforeEach public void setUp() { ASM_1 = inputParameters.getRemappedFrom(); ASM_2 = inputParameters.getAssemblyAccession(); @@ -182,7 +169,7 @@ public void setUp() { clusteringWriterPostMergeAndSplit.setJobExecution(jobExecution); } - @After + @AfterEach public void tearDown() { mongoTemplate.getDb().drop(); } @@ -191,7 +178,7 @@ public void tearDown() { @DirtiesContext public void test_variant_clustered_rs_id_does_not_exist() throws Exception { SubmittedVariantEntity submittedVariantEntity1 = getSubmittedVariantEntity(ASM_1, PROJECT_1, 1000, 3000000001L, - 5000000000L, NOT_REMAPPED); + 5000000000L, NOT_REMAPPED); mongoTemplate.insert(submittedVariantEntity1, SUBMITTED_VARIANT_COLLECTION); ClusteredVariantEntity clusteredVariantEntity1 = getClusteredVariantEntity(ASM_1, 1000, 3000000001L); @@ -199,7 +186,7 @@ public void test_variant_clustered_rs_id_does_not_exist() throws Exception { //variant remapped from asm1 to asm2 SubmittedVariantEntity submittedVariantEntity2 = getSubmittedVariantEntity(ASM_2, PROJECT_1, 2000, 3000000001L, - 5000000000L, ASM_1); + 5000000000L, ASM_1); mongoTemplate.insert(submittedVariantEntity2, SUBMITTED_VARIANT_COLLECTION); //asm2 clustered @@ -209,9 +196,9 @@ public void test_variant_clustered_rs_id_does_not_exist() throws Exception { //get all submitted variants with assembly asm2 and assert rs id List submittedVariants = mongoTemplate.findAll(SubmittedVariantEntity.class).stream() - .filter(s -> s.getReferenceSequenceAccession() - .equals(ASM_2)) - .collect(Collectors.toList()); + .filter(s -> s.getReferenceSequenceAccession() + .equals(ASM_2)) + .collect(Collectors.toList()); assertEquals(1, submittedVariants.size()); assertEquals(3000000001L, submittedVariants.get(0).getClusteredVariantAccession().longValue()); @@ -221,9 +208,9 @@ public void test_variant_clustered_rs_id_does_not_exist() throws Exception { assertEquals(3000000001L, clusteredVariants.get(0).getAccession().longValue()); assertEquals(3000000001L, clusteredVariants.get(1).getAccession().longValue()); assertEquals(ASM_1, clusteredVariants.stream().filter(s -> s.getStart() == 1000).collect(Collectors.toList()) - .get(0).getAssemblyAccession()); + .get(0).getAssemblyAccession()); assertEquals(ASM_2, clusteredVariants.stream().filter(s -> s.getStart() == 2000).collect(Collectors.toList()) - .get(0).getAssemblyAccession()); + .get(0).getAssemblyAccession()); //assert no submitted variant operations should happen List submittedVariantOperations = mongoTemplate.findAll( @@ -240,10 +227,10 @@ public void test_variant_clustered_rs_id_does_not_exist() throws Exception { @DirtiesContext public void test_variant_clustered_rs_id_exist() throws Exception { SubmittedVariantEntity submittedVariantEntity1 = getSubmittedVariantEntity(ASM_1, PROJECT_1, 1000, 3000000000L, - 5000000000L, NOT_REMAPPED); + 5000000000L, NOT_REMAPPED); mongoTemplate.insert(submittedVariantEntity1, SUBMITTED_VARIANT_COLLECTION); SubmittedVariantEntity submittedVariantEntity2 = getSubmittedVariantEntity(ASM_2, PROJECT_2, 2000, 3000000006L, - 5000000006L, NOT_REMAPPED); + 5000000006L, NOT_REMAPPED); mongoTemplate.insert(submittedVariantEntity2, SUBMITTED_VARIANT_COLLECTION); ClusteredVariantEntity clusteredVariantEntity1 = getClusteredVariantEntity(ASM_1, 1000, 3000000000L); @@ -253,7 +240,7 @@ public void test_variant_clustered_rs_id_exist() throws Exception { //variant remapped from asm1 to asm 2 SubmittedVariantEntity submittedVariantEntity3 = getSubmittedVariantEntity(ASM_2, PROJECT_1, 2000, 3000000000L, - 5000000000L, ASM_1); + 5000000000L, ASM_1); mongoTemplate.insert(submittedVariantEntity3, SUBMITTED_VARIANT_COLLECTION); //asm2 clustered @@ -264,18 +251,18 @@ public void test_variant_clustered_rs_id_exist() throws Exception { //get all submitted variants with assembly asm2 and assert rs id List submittedVariants = mongoTemplate.findAll(SubmittedVariantEntity.class).stream() - .filter(s -> s.getReferenceSequenceAccession() - .equals(ASM_2)) - .collect(Collectors.toList()); + .filter(s -> s.getReferenceSequenceAccession() + .equals(ASM_2)) + .collect(Collectors.toList()); assertEquals(2, submittedVariants.size()); //get all clusteredVariantEntity and check rs id for all List clusteredVariants = mongoTemplate.findAll(ClusteredVariantEntity.class); assertEquals(2, clusteredVariants.size()); List clusteredVariantsASM_1 = clusteredVariants.stream() - .filter(cve->cve.getAssemblyAccession().equals(ASM_1)).collect(Collectors.toList()); + .filter(cve -> cve.getAssemblyAccession().equals(ASM_1)).collect(Collectors.toList()); List clusteredVariantsASM_2 = clusteredVariants.stream() - .filter(cve->cve.getAssemblyAccession().equals(ASM_2)).collect(Collectors.toList()); + .filter(cve -> cve.getAssemblyAccession().equals(ASM_2)).collect(Collectors.toList()); assertEquals(1, clusteredVariantsASM_1.size()); assertEquals(1, clusteredVariantsASM_2.size()); assertEquals(3000000000L, clusteredVariantsASM_1.get(0).getAccession().longValue()); @@ -290,8 +277,8 @@ public void test_variant_clustered_rs_id_exist() throws Exception { SubmittedVariantOperationEntity submittedVariantOperationEntity = submittedVariantOperationEntities.get(0); assertEquals(5000000006L, submittedVariantOperationEntity.getAccession().longValue()); assertEquals(3000000006L, - submittedVariantOperationEntity.getInactiveObjects().get(0).getClusteredVariantAccession() - .longValue()); + submittedVariantOperationEntity.getInactiveObjects().get(0).getClusteredVariantAccession() + .longValue()); assertEquals(EventType.UPDATED, submittedVariantOperationEntity.getEventType()); } @@ -299,12 +286,12 @@ public void test_variant_clustered_rs_id_exist() throws Exception { @DirtiesContext public void test_variant_not_clustered_rs_id_does_not_exist() throws Exception { SubmittedVariantEntity submittedVariantEntity1 = getSubmittedVariantEntity(ASM_1, PROJECT_1, 1000, null, - 5000000000L, NOT_REMAPPED); + 5000000000L, NOT_REMAPPED); mongoTemplate.insert(submittedVariantEntity1, SUBMITTED_VARIANT_COLLECTION); //variant remapped from asm1 to asm2 SubmittedVariantEntity submittedVariantEntity2 = getSubmittedVariantEntity(ASM_2, PROJECT_1, 2000, null, - 5000000000L, ASM_1); + 5000000000L, ASM_1); mongoTemplate.insert(submittedVariantEntity2, SUBMITTED_VARIANT_COLLECTION); //asm2 clustered @@ -320,13 +307,13 @@ public void test_variant_not_clustered_rs_id_does_not_exist() throws Exception { assertEquals(new HashSet<>(Collections.singletonList(3000000000L)), submittedVariants.stream().map(SubmittedVariantEntity::getClusteredVariantAccession) .filter(Objects::nonNull).collect(Collectors.toSet()) - ); + ); assertEquals(new HashSet<>(Collections.singletonList(3000000000L)), submittedVariants.stream().map(SubmittedVariantEntity::getBackPropagatedVariantAccession) .filter(Objects::nonNull).collect(Collectors.toSet()) ); assertTrue(submittedVariants.stream().map(SubmittedVariantEntity::getReferenceSequenceAccession).collect( - Collectors.toList()).containsAll(Arrays.asList(ASM_1, ASM_2))); + Collectors.toList()).containsAll(Arrays.asList(ASM_1, ASM_2))); //get all clusteredVariantEntity and check rs id for all List clusteredVariants = mongoTemplate.findAll(ClusteredVariantEntity.class); @@ -347,25 +334,25 @@ public void test_variant_not_clustered_rs_id_does_not_exist() throws Exception { .collect(Collectors.toList()).containsAll(Arrays.asList(EventType.UPDATED, EventType.RS_BACK_PROPAGATED))); assertEquals("Clustering submitted variant 5000000000 with rs3000000000", - submittedVariantOperationEntities.stream().filter(svoe -> svoe.getInactiveObjects().get(0) - .getReferenceSequenceAccession() - .equals(ASM_2)).findFirst().get() - .getReason()); + submittedVariantOperationEntities.stream().filter(svoe -> svoe.getInactiveObjects().get(0) + .getReferenceSequenceAccession() + .equals(ASM_2)).findFirst().get() + .getReason()); assertEquals("Back-propagating rs3000000000 for submitted variant ss5000000000 after remapping to asm2.", - submittedVariantOperationEntities.stream().filter(svoe -> svoe.getInactiveObjects().get(0) - .getReferenceSequenceAccession() - .equals(ASM_1)).findFirst().get() - .getReason()); + submittedVariantOperationEntities.stream().filter(svoe -> svoe.getInactiveObjects().get(0) + .getReferenceSequenceAccession() + .equals(ASM_1)).findFirst().get() + .getReason()); } @Test @DirtiesContext public void test_variant_not_clustered_rs_id_exist() throws Exception { SubmittedVariantEntity submittedVariantEntity1 = getSubmittedVariantEntity(ASM_1, PROJECT_1, 1000, null, - 5000000000L, NOT_REMAPPED); + 5000000000L, NOT_REMAPPED); mongoTemplate.insert(submittedVariantEntity1, SUBMITTED_VARIANT_COLLECTION); SubmittedVariantEntity submittedVariantEntity2 = getSubmittedVariantEntity(ASM_2, PROJECT_2, 2000, 3000000006L, - 5000000006L, NOT_REMAPPED); + 5000000006L, NOT_REMAPPED); mongoTemplate.insert(submittedVariantEntity2, SUBMITTED_VARIANT_COLLECTION); ClusteredVariantEntity clusteredVariantEntity2 = getClusteredVariantEntity(ASM_2, 2000, 3000000006L); @@ -373,7 +360,7 @@ public void test_variant_not_clustered_rs_id_exist() throws Exception { //variant remapped from asm1 to asm2 SubmittedVariantEntity submittedVariantEntity3 = getSubmittedVariantEntity(ASM_2, PROJECT_1, 2000, null, - 5000000000L, ASM_1); + 5000000000L, ASM_1); mongoTemplate.insert(submittedVariantEntity3, SUBMITTED_VARIANT_COLLECTION); //asm2 clustered @@ -389,7 +376,7 @@ public void test_variant_not_clustered_rs_id_exist() throws Exception { // submittedVariantEntity1 will receive the back-propagated RS assertEquals(submittedVariants.stream().map(SubmittedVariantEntity::getClusteredVariantAccession) .filter(Objects::nonNull).collect(Collectors.toSet()), - new HashSet<>(Collections.singletonList(3000000006L))); + new HashSet<>(Collections.singletonList(3000000006L))); assertEquals(submittedVariants.stream().map(SubmittedVariantEntity::getBackPropagatedVariantAccession) .filter(Objects::nonNull).collect(Collectors.toSet()), new HashSet<>(Collections.singletonList(3000000006L))); @@ -417,41 +404,41 @@ public void test_variant_not_clustered_rs_id_exist() throws Exception { .collect(Collectors.toList()).containsAll(Arrays.asList(EventType.UPDATED, EventType.RS_BACK_PROPAGATED))); assertEquals("Clustering submitted variant 5000000000 with rs3000000006", - submittedVariantOperationEntities.stream().filter(svoe -> svoe.getInactiveObjects().get(0) - .getReferenceSequenceAccession() - .equals(ASM_2)).findFirst().get() - .getReason()); + submittedVariantOperationEntities.stream().filter(svoe -> svoe.getInactiveObjects().get(0) + .getReferenceSequenceAccession() + .equals(ASM_2)).findFirst().get() + .getReason()); assertEquals("Back-propagating rs3000000006 for submitted variant ss5000000000 after remapping to asm2.", - submittedVariantOperationEntities.stream().filter(svoe -> svoe.getInactiveObjects().get(0) - .getReferenceSequenceAccession() - .equals(ASM_1)).findFirst().get() - .getReason()); + submittedVariantOperationEntities.stream().filter(svoe -> svoe.getInactiveObjects().get(0) + .getReferenceSequenceAccession() + .equals(ASM_1)).findFirst().get() + .getReason()); } /** * This test is for the testing the detection of Merge and RS split candidates. * These are already clustered variants, but due to remapping, the variants with same rs id might end up in two * different locations or different types. These needs to be identified for rectifying later. - * + *

* --------------------Before Clustering-------------------- - * - * SubmittedVariantEntity + *

+ * SubmittedVariantEntity * SS RS ASM STUDY CONTIG POS REF ALT * 500 306 ASM1 PRJEB1 Chr1 1000 A T (original) * 501 306 ASM1 PRJEB2 Chr1 1000 A T (original) * 500 306 ASM2 PRJEB1 Chr1 1000 A T (remapped) * 501 306 ASM2 PRJEB2 Chr1 1500 A T (remapped) - * - * ClusteredVariantEntity + *

+ * ClusteredVariantEntity * RS HASH ASM POS CONTIG TYPE * 306 ASM1_Chr1_1000_SNV ASM1 1000 Chr1 SNV - * + *

* SS id 500 and 501 has same RS because of remapping, but they are now at different positions and can't have same RS id. * These needs to be identified and stored in submittedVariantOperationEntity table for rectification. - * + *

* --------------------After Detection-------------------- - * - * SubmittedVariantOperationEntity + *

+ * SubmittedVariantOperationEntity * ACCESSION EVENT_TYPE REASON INACTIVe_OBJECTS * 306 RS_SPLIT_CANDIDATE Hash mismatch with 306 {ss-500 and ss-501} */ @@ -460,68 +447,68 @@ public void test_variant_not_clustered_rs_id_exist() throws Exception { public void test_detect_RS_MERGE_AND_SPLIT_different_batches() throws Exception { //Batch 1 SubmittedVariantEntity submittedVariantEntityRemapped1 = getSubmittedVariantEntity(ASM_2, "project_1", - 5000000001L, 3000000001L, 1000,"A", "T", ASM_1); + 5000000001L, 3000000001L, 1000, "A", "T", ASM_1); mongoTemplate.insert(submittedVariantEntityRemapped1, SUBMITTED_VARIANT_COLLECTION); SubmittedVariantEntity submittedVariantEntityRemapped2 = getSubmittedVariantEntity(ASM_2, "project_2", - 5000000002L, 3000000003L, 2000,"A", "T", ASM_1); + 5000000002L, 3000000003L, 2000, "A", "T", ASM_1); mongoTemplate.insert(submittedVariantEntityRemapped2, SUBMITTED_VARIANT_COLLECTION); - clusteringWriterPreMergeAndSplit.write(Arrays.asList(submittedVariantEntityRemapped1, submittedVariantEntityRemapped2)); + clusteringWriterPreMergeAndSplit.write(Chunk.of(submittedVariantEntityRemapped1, submittedVariantEntityRemapped2)); //Batch 2 - split event SubmittedVariantEntity submittedVariantEntityRemapped3 = getSubmittedVariantEntity(ASM_2, "project_3", - 5000000003L, 3000000001L, 3000,"A", "T", ASM_1); + 5000000003L, 3000000001L, 3000, "A", "T", ASM_1); mongoTemplate.insert(submittedVariantEntityRemapped3, SUBMITTED_VARIANT_COLLECTION); - clusteringWriterPreMergeAndSplit.write(Arrays.asList(submittedVariantEntityRemapped3)); + clusteringWriterPreMergeAndSplit.write(Chunk.of(submittedVariantEntityRemapped3)); //Batch 3 - split event SubmittedVariantEntity submittedVariantEntityRemapped4 = getSubmittedVariantEntity(ASM_2, "project_4", - 5000000004L, 3000000001L, 4000,"A", "T", ASM_1); + 5000000004L, 3000000001L, 4000, "A", "T", ASM_1); mongoTemplate.insert(submittedVariantEntityRemapped4, SUBMITTED_VARIANT_COLLECTION); - clusteringWriterPreMergeAndSplit.write(Arrays.asList(submittedVariantEntityRemapped4)); + clusteringWriterPreMergeAndSplit.write(Chunk.of(submittedVariantEntityRemapped4)); //Batch 4 - merge event SubmittedVariantEntity submittedVariantEntityRemapped5 = getSubmittedVariantEntity(ASM_2, "project_5", - 5000000005L, 3000000002L, 2000,"A", "T", ASM_1); + 5000000005L, 3000000002L, 2000, "A", "T", ASM_1); mongoTemplate.insert(submittedVariantEntityRemapped5, SUBMITTED_VARIANT_COLLECTION); - clusteringWriterPreMergeAndSplit.write(Arrays.asList(submittedVariantEntityRemapped5)); + clusteringWriterPreMergeAndSplit.write(Chunk.of(submittedVariantEntityRemapped5)); //Batch 5 - merge event SubmittedVariantEntity submittedVariantEntityRemapped6 = getSubmittedVariantEntity(ASM_2, "project_6", - 5000000006L, 3000000002L, 2000,"A", "T", ASM_1); + 5000000006L, 3000000002L, 2000, "A", "T", ASM_1); mongoTemplate.insert(submittedVariantEntityRemapped6, SUBMITTED_VARIANT_COLLECTION); - clusteringWriterPreMergeAndSplit.write(Arrays.asList(submittedVariantEntityRemapped6)); + clusteringWriterPreMergeAndSplit.write(Chunk.of(submittedVariantEntityRemapped6)); //Assert assertMergeAndSplitCandidatesScenarios(Arrays.asList(submittedVariantEntityRemapped1, - submittedVariantEntityRemapped2,submittedVariantEntityRemapped3,submittedVariantEntityRemapped4, - submittedVariantEntityRemapped5,submittedVariantEntityRemapped6)); + submittedVariantEntityRemapped2, submittedVariantEntityRemapped3, submittedVariantEntityRemapped4, + submittedVariantEntityRemapped5, submittedVariantEntityRemapped6)); } @Test @DirtiesContext public void test_detect_RS_MERGE_AND_SPLIT_same_batch() throws Exception { SubmittedVariantEntity submittedVariantEntityRemapped1 = getSubmittedVariantEntity(ASM_2, "project_1", - 5000000001L, 3000000001L, 1000,"A", "T", ASM_1); + 5000000001L, 3000000001L, 1000, "A", "T", ASM_1); mongoTemplate.insert(submittedVariantEntityRemapped1, SUBMITTED_VARIANT_COLLECTION); SubmittedVariantEntity submittedVariantEntityRemapped2 = getSubmittedVariantEntity(ASM_2, "project_2", - 5000000002L, 3000000003L, 2000,"A", "T", ASM_1); + 5000000002L, 3000000003L, 2000, "A", "T", ASM_1); mongoTemplate.insert(submittedVariantEntityRemapped2, SUBMITTED_VARIANT_COLLECTION); SubmittedVariantEntity submittedVariantEntityRemapped3 = getSubmittedVariantEntity(ASM_2, "project_3", - 5000000003L, 3000000001L, 3000,"A", "T", ASM_1); + 5000000003L, 3000000001L, 3000, "A", "T", ASM_1); mongoTemplate.insert(submittedVariantEntityRemapped3, SUBMITTED_VARIANT_COLLECTION); SubmittedVariantEntity submittedVariantEntityRemapped4 = getSubmittedVariantEntity(ASM_2, "project_4", - 5000000004L, 3000000001L, 4000,"A", "T", ASM_1); + 5000000004L, 3000000001L, 4000, "A", "T", ASM_1); mongoTemplate.insert(submittedVariantEntityRemapped4, SUBMITTED_VARIANT_COLLECTION); SubmittedVariantEntity submittedVariantEntityRemapped5 = getSubmittedVariantEntity(ASM_2, "project_5", - 5000000005L, 3000000002L, 2000,"A", "T", ASM_1); + 5000000005L, 3000000002L, 2000, "A", "T", ASM_1); mongoTemplate.insert(submittedVariantEntityRemapped5, SUBMITTED_VARIANT_COLLECTION); SubmittedVariantEntity submittedVariantEntityRemapped6 = getSubmittedVariantEntity(ASM_2, "project_6", - 5000000006L, 3000000002L, 2000,"A", "T", ASM_1); + 5000000006L, 3000000002L, 2000, "A", "T", ASM_1); mongoTemplate.insert(submittedVariantEntityRemapped6, SUBMITTED_VARIANT_COLLECTION); List submittedVariantEntityList = new ArrayList<>(); @@ -531,44 +518,44 @@ public void test_detect_RS_MERGE_AND_SPLIT_same_batch() throws Exception { submittedVariantEntityList.add(submittedVariantEntityRemapped4); submittedVariantEntityList.add(submittedVariantEntityRemapped5); submittedVariantEntityList.add(submittedVariantEntityRemapped6); - clusteringWriterPreMergeAndSplit.write(Arrays.asList(submittedVariantEntityRemapped1, submittedVariantEntityRemapped2, - submittedVariantEntityRemapped3, submittedVariantEntityRemapped4, - submittedVariantEntityRemapped5, submittedVariantEntityRemapped6)); + clusteringWriterPreMergeAndSplit.write(Chunk.of(submittedVariantEntityRemapped1, submittedVariantEntityRemapped2, + submittedVariantEntityRemapped3, submittedVariantEntityRemapped4, + submittedVariantEntityRemapped5, submittedVariantEntityRemapped6)); //Assert assertMergeAndSplitCandidatesScenarios(Arrays.asList(submittedVariantEntityRemapped1, - submittedVariantEntityRemapped2,submittedVariantEntityRemapped3,submittedVariantEntityRemapped4, - submittedVariantEntityRemapped5,submittedVariantEntityRemapped6)); + submittedVariantEntityRemapped2, submittedVariantEntityRemapped3, submittedVariantEntityRemapped4, + submittedVariantEntityRemapped5, submittedVariantEntityRemapped6)); } - private void assertMergeAndSplitCandidatesScenarios(List submittedVariantEntities){ + private void assertMergeAndSplitCandidatesScenarios(List submittedVariantEntities) { //assert clusteredVariantEntity List clusteredVariantEntities = mongoTemplate.findAll(ClusteredVariantEntity.class); assertEquals(4, clusteredVariantEntities.size()); - assertEquals(1,clusteredVariantEntities.stream().filter(cve->cve.getAccession().equals(3000000003L)).count()); - assertEquals(3,clusteredVariantEntities.stream().filter(cve->cve.getAccession().equals(3000000001L)).count()); + assertEquals(1, clusteredVariantEntities.stream().filter(cve -> cve.getAccession().equals(3000000003L)).count()); + assertEquals(3, clusteredVariantEntities.stream().filter(cve -> cve.getAccession().equals(3000000001L)).count()); //assert merge event SubmittedVariantOperationEntity submittedVariantOperationEntityMerge = mongoTemplate.findAll(SubmittedVariantOperationEntity.class).stream() - .filter(svoe->svoe.getEventType().equals(EventType.RS_MERGE_CANDIDATES)) + .filter(svoe -> svoe.getEventType().equals(EventType.RS_MERGE_CANDIDATES)) .collect(Collectors.toList()).get(0); assertEquals(EventType.RS_MERGE_CANDIDATES, submittedVariantOperationEntityMerge.getEventType()); - assertEquals(3000000003L,submittedVariantOperationEntityMerge.getAccession().longValue()); + assertEquals(3000000003L, submittedVariantOperationEntityMerge.getAccession().longValue()); assertEquals("RS mismatch with 3000000003", submittedVariantOperationEntityMerge.getReason()); assertEquals(3, submittedVariantOperationEntityMerge.getInactiveObjects().size()); assertTrue(submittedVariantOperationEntityMerge.getInactiveObjects().containsAll( Arrays.asList(new SubmittedVariantInactiveEntity(submittedVariantEntities.get(1)), new SubmittedVariantInactiveEntity(submittedVariantEntities.get(4)), - new SubmittedVariantInactiveEntity(submittedVariantEntities.get(5))))); + new SubmittedVariantInactiveEntity(submittedVariantEntities.get(5))))); //assert split event SubmittedVariantOperationEntity submittedVariantOperationEntitySplit = mongoTemplate.findAll(SubmittedVariantOperationEntity.class).stream() - .filter(svoe->svoe.getEventType().equals(EventType.RS_SPLIT_CANDIDATES)) + .filter(svoe -> svoe.getEventType().equals(EventType.RS_SPLIT_CANDIDATES)) .collect(Collectors.toList()).get(0); assertEquals(EventType.RS_SPLIT_CANDIDATES, submittedVariantOperationEntitySplit.getEventType()); - assertEquals(3000000001L,submittedVariantOperationEntitySplit.getAccession().longValue()); + assertEquals(3000000001L, submittedVariantOperationEntitySplit.getAccession().longValue()); assertEquals("Hash mismatch with 3000000001", submittedVariantOperationEntitySplit.getReason()); assertEquals(3, submittedVariantOperationEntitySplit.getInactiveObjects().size()); assertTrue(submittedVariantOperationEntitySplit.getInactiveObjects().containsAll( @@ -580,7 +567,7 @@ private void assertMergeAndSplitCandidatesScenarios(List private SubmittedVariantEntity getSubmittedVariantEntity(String assembly, String project, long ss, Long rs, long start, String ref, String alt, String remappedFrom) { SubmittedVariant submittedClustered = new SubmittedVariant(assembly, TAXONOMY, project, CONTIG, start, ref, alt, - rs); + rs); String hash1 = hashingFunction.apply(submittedClustered); return new SubmittedVariantEntity(ss, hash1, submittedClustered, 1, remappedFrom, LocalDateTime.now(), null); } @@ -602,41 +589,41 @@ private ClusteredVariantEntity getClusteredVariantEntity(String assembly, long s private void clusterVariants(List submittedVariantEntities) throws Exception { - clusteringWriterPreMergeAndSplit.write(submittedVariantEntities); + clusteringWriterPreMergeAndSplit.write(new Chunk<>(submittedVariantEntities)); List mergeCandidates = new ArrayList<>(); List splitCandidates = new ArrayList<>(); SubmittedVariantOperationEntity tempSVO; rsMergeCandidatesReader.open(new ExecutionContext()); - while((tempSVO = rsMergeCandidatesReader.read()) != null) { + while ((tempSVO = rsMergeCandidatesReader.read()) != null) { mergeCandidates.add(tempSVO); } rsSplitCandidatesReader.open(new ExecutionContext()); - while((tempSVO = rsSplitCandidatesReader.read()) != null) { + while ((tempSVO = rsSplitCandidatesReader.read()) != null) { splitCandidates.add(tempSVO); } - rsMergeWriter.write(mergeCandidates); - rsSplitWriter.write(splitCandidates); + rsMergeWriter.write(new Chunk<>(mergeCandidates)); + rsSplitWriter.write(new Chunk<>(splitCandidates)); ClusteringMongoReader unclusteredVariantsReader = new ClusteringMongoReader(this.mongoTemplate, ASM_2, 100, - false); + false); unclusteredVariantsReader.initializeReader(); List unclusteredVariants = new ArrayList<>(); SubmittedVariantEntity tempSV; - while((tempSV = unclusteredVariantsReader.read()) != null) { + while ((tempSV = unclusteredVariantsReader.read()) != null) { unclusteredVariants.add(tempSV); } unclusteredVariantsReader.close(); // Cluster non-clustered variants - clusteringWriterPostMergeAndSplit.write(unclusteredVariants); + clusteringWriterPostMergeAndSplit.write(new Chunk<>(unclusteredVariants)); // Spring has a mandatory requirement of even small functionality being writers. // To satisfy that, we pass in a dummy object to invoke the writer // which basically clears the merge and split operations after they were processed above - clearRSMergeAndSplitCandidates.write(Collections.singletonList(new Object())); + clearRSMergeAndSplitCandidates.write(Chunk.of(new Object())); ClusteringMongoReader originalAssemblyUnclusteredVariantsReader = new ClusteringMongoReader(this.mongoTemplate, ASM_1, 100, false); originalAssemblyUnclusteredVariantsReader.initializeReader(); - while((tempSV = originalAssemblyUnclusteredVariantsReader.read()) != null) { - backPropagatedRSWriter.write(Collections.singletonList(tempSV)); + while ((tempSV = originalAssemblyUnclusteredVariantsReader.read()) != null) { + backPropagatedRSWriter.write(Chunk.of(tempSV)); } } } diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/clustering_writer/ReuseAccessionClusteringWriterTest.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/clustering_writer/ReuseAccessionClusteringWriterTest.java index 6784e3842..c1b132875 100644 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/clustering_writer/ReuseAccessionClusteringWriterTest.java +++ b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/clustering_writer/ReuseAccessionClusteringWriterTest.java @@ -15,15 +15,13 @@ */ package uk.ac.ebi.eva.accession.clustering.batch.io.clustering_writer; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mockito; import org.springframework.batch.core.JobExecution; +import org.springframework.batch.item.Chunk; import org.springframework.batch.item.ExecutionContext; import org.springframework.batch.item.ItemWriter; import org.springframework.beans.factory.annotation.Autowired; @@ -36,17 +34,15 @@ import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; - +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.ampt2d.commons.accession.hashing.SHA1HashingFunction; - import uk.ac.ebi.eva.accession.clustering.batch.io.ClusteringMongoReader; import uk.ac.ebi.eva.accession.clustering.batch.io.ClusteringWriter; import uk.ac.ebi.eva.accession.clustering.batch.io.RSSplitWriter; import uk.ac.ebi.eva.accession.clustering.metric.ClusteringMetric; import uk.ac.ebi.eva.accession.clustering.parameters.InputParameters; import uk.ac.ebi.eva.accession.clustering.test.configuration.BatchTestConfiguration; -import uk.ac.ebi.eva.accession.clustering.test.rule.FixSpringMongoDbRule; +import uk.ac.ebi.eva.accession.clustering.test.configuration.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.ClusteredVariantAccessioningConfiguration; import uk.ac.ebi.eva.accession.core.model.ClusteredVariant; import uk.ac.ebi.eva.accession.core.model.IClusteredVariant; @@ -61,6 +57,7 @@ import uk.ac.ebi.eva.accession.core.service.nonhuman.ClusteredVariantAccessioningService; import uk.ac.ebi.eva.accession.core.summary.ClusteredVariantSummaryFunction; import uk.ac.ebi.eva.accession.core.summary.SubmittedVariantSummaryFunction; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.commons.core.models.VariantType; import uk.ac.ebi.eva.commons.mongodb.readers.MongoDbCursorItemReader; import uk.ac.ebi.eva.metrics.metric.MetricCompute; @@ -70,8 +67,8 @@ import java.util.List; import java.util.function.Function; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.springframework.data.mongodb.core.query.Criteria.where; import static uk.ac.ebi.eva.accession.clustering.batch.io.clustering_writer.ClusteringAssertions.assertClusteringCounts; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.BACK_PROPAGATED_RS_WRITER; @@ -85,19 +82,17 @@ /** * This class handles some scenarios of ClusteringWriter where an existing RS is reused. - * + *

* The scenarios tested here are those about either reusing the clustered variant accessions (RSs) that a submitted * variant provides, or reusing an RS that exists in the database because it's equivalent (shares identifying fields). * Other test classes in this folder take care of other scenarios. */ -@RunWith(SpringRunner.class) +@ExtendWith(SpringExtension.class) @EnableAutoConfiguration -@ContextConfiguration(classes = {ClusteredVariantAccessioningConfiguration.class, BatchTestConfiguration.class}) +@ContextConfiguration(classes = {ClusteredVariantAccessioningConfiguration.class, BatchTestConfiguration.class, + MongoTestConfiguration.class}) @TestPropertySource("classpath:clustering-writer-test.properties") -public class ReuseAccessionClusteringWriterTest { - - private static final String TEST_DB = "test-db"; - +public class ReuseAccessionClusteringWriterTest extends MongoTestContainerHelper { private static String ASM_1; private static String ASM_2; @@ -160,11 +155,7 @@ public class ReuseAccessionClusteringWriterTest { private Function clusteredHashingFunction; - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName(TEST_DB).build()); - - @Before + @BeforeEach public void setUp() { ASM_1 = inputParameters.getRemappedFrom(); ASM_2 = inputParameters.getAssemblyAccession(); @@ -179,7 +170,7 @@ public void setUp() { clusteringWriterPostMergeAndSplit.setJobExecution(jobExecution); } - @After + @AfterEach public void tearDown() { mongoTemplate.getDb().drop(); metricCompute.clearCount(); @@ -233,11 +224,11 @@ private DbsnpSubmittedVariantEntity createDbsnpSubmittedVariantEntity(String ass SubmittedVariant submittedClustered = new SubmittedVariant(assembly, 1000, "project", "1", 100L, "T", "A", rs); String hash1 = hashingFunction.apply(submittedClustered); DbsnpSubmittedVariantEntity dbsnpSubmittedVariantEntity = new DbsnpSubmittedVariantEntity(ss, hash1, - submittedClustered, - 1); + submittedClustered, + 1); dbsnpSubmittedVariantEntity.setRemappedFrom(remappedFromAssembly); mongoTemplate.save(dbsnpSubmittedVariantEntity, - mongoTemplate.getCollectionName(DbsnpSubmittedVariantEntity.class)); + mongoTemplate.getCollectionName(DbsnpSubmittedVariantEntity.class)); return dbsnpSubmittedVariantEntity; } @@ -308,41 +299,41 @@ public void reuse_dbsnp_clustered_accession_when_clustering_an_eva_submitted_var private void clusterVariants(List submittedVariantEntities) throws Exception { - clusteringWriterPreMergeAndSplit.write(submittedVariantEntities); + clusteringWriterPreMergeAndSplit.write(new Chunk<>(submittedVariantEntities)); List mergeCandidates = new ArrayList<>(); List splitCandidates = new ArrayList<>(); SubmittedVariantOperationEntity tempSVO; rsMergeCandidatesReader.open(new ExecutionContext()); - while((tempSVO = rsMergeCandidatesReader.read()) != null) { + while ((tempSVO = rsMergeCandidatesReader.read()) != null) { mergeCandidates.add(tempSVO); } rsSplitCandidatesReader.open(new ExecutionContext()); - while((tempSVO = rsSplitCandidatesReader.read()) != null) { + while ((tempSVO = rsSplitCandidatesReader.read()) != null) { splitCandidates.add(tempSVO); } - rsMergeWriter.write(mergeCandidates); - rsSplitWriter.write(splitCandidates); + rsMergeWriter.write(new Chunk<>(mergeCandidates)); + rsSplitWriter.write(new Chunk<>(splitCandidates)); ClusteringMongoReader unclusteredVariantsReader = new ClusteringMongoReader(this.mongoTemplate, ASM_2, 100, - false); + false); unclusteredVariantsReader.initializeReader(); List unclusteredVariants = new ArrayList<>(); SubmittedVariantEntity tempSV; - while((tempSV = unclusteredVariantsReader.read()) != null) { + while ((tempSV = unclusteredVariantsReader.read()) != null) { unclusteredVariants.add(tempSV); } unclusteredVariantsReader.close(); // Cluster non-clustered variants - clusteringWriterPostMergeAndSplit.write(unclusteredVariants); + clusteringWriterPostMergeAndSplit.write(new Chunk<>(unclusteredVariants)); // Spring has a mandatory requirement of even small functionality being writers. // To satisfy that, we pass in a dummy object to invoke the writer // which basically clears the merge and split operations after they were processed above - clearRSMergeAndSplitCandidates.write(Collections.singletonList(new Object())); + clearRSMergeAndSplitCandidates.write(Chunk.of(new Object())); ClusteringMongoReader originalAssemblyUnclusteredVariantsReader = new ClusteringMongoReader(this.mongoTemplate, ASM_1, 100, false); originalAssemblyUnclusteredVariantsReader.initializeReader(); - while((tempSV = originalAssemblyUnclusteredVariantsReader.read()) != null) { - backPropagatedRSWriter.write(Collections.singletonList(tempSV)); + while ((tempSV = originalAssemblyUnclusteredVariantsReader.read()) != null) { + backPropagatedRSWriter.write(Chunk.of(tempSV)); } } } diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/processors/VariantToSubmittedVariantEntityProcessorTest.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/processors/VariantToSubmittedVariantEntityProcessorTest.java index fb7b037db..b2af4d2e8 100644 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/processors/VariantToSubmittedVariantEntityProcessorTest.java +++ b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/processors/VariantToSubmittedVariantEntityProcessorTest.java @@ -15,8 +15,8 @@ */ package uk.ac.ebi.eva.accession.clustering.batch.processors; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import uk.ac.ebi.ampt2d.commons.accession.hashing.SHA1HashingFunction; import uk.ac.ebi.eva.accession.core.model.ISubmittedVariant; import uk.ac.ebi.eva.accession.core.model.SubmittedVariant; @@ -28,8 +28,8 @@ import java.util.function.Function; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThrows; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; public class VariantToSubmittedVariantEntityProcessorTest { @@ -43,7 +43,7 @@ public class VariantToSubmittedVariantEntityProcessorTest { private VariantToSubmittedVariantEntityProcessor processor; - @Before + @BeforeEach public void setUp() { processor = new VariantToSubmittedVariantEntityProcessor(ASSEMBLY_ACCESSION); hashingFunction = new SubmittedVariantSummaryFunction().andThen(new SHA1HashingFunction()); diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/ClusteringVariantJobConfigurationTest.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/ClusteringVariantJobConfigurationTest.java index 391aada9e..f9fc8abac 100644 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/ClusteringVariantJobConfigurationTest.java +++ b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/ClusteringVariantJobConfigurationTest.java @@ -15,40 +15,38 @@ */ package uk.ac.ebi.eva.accession.clustering.configuration.batch.jobs; -import com.lordofthejars.nosqlunit.annotation.UsingDataSet; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.batch.core.BatchStatus; import org.springframework.batch.core.JobExecution; import org.springframework.batch.core.StepExecution; import org.springframework.batch.test.JobLauncherTestUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.context.ApplicationContext; +import org.springframework.core.io.ResourceLoader; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.http.HttpMethod; import org.springframework.http.HttpStatus; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import org.springframework.test.web.client.ExpectedCount; import org.springframework.test.web.client.MockRestServiceServer; - import org.springframework.web.client.RestTemplate; import uk.ac.ebi.eva.accession.clustering.configuration.batch.io.RSMergeAndSplitCandidatesReaderConfiguration; import uk.ac.ebi.eva.accession.clustering.parameters.InputParameters; import uk.ac.ebi.eva.accession.clustering.test.DatabaseState; +import uk.ac.ebi.eva.accession.clustering.test.configuration.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.accession.clustering.test.configuration.BatchTestConfiguration; -import uk.ac.ebi.eva.accession.clustering.test.rule.FixSpringMongoDbRule; +import uk.ac.ebi.eva.accession.clustering.test.configuration.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantInactiveEntity; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantOperationEntity; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; +import uk.ac.ebi.eva.accession.core.utils.MongoTestDataLoader; import uk.ac.ebi.eva.metrics.count.CountServiceParameters; import java.net.URI; @@ -58,7 +56,7 @@ import java.util.List; import java.util.stream.Collectors; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.springframework.test.web.client.match.MockRestRequestMatchers.method; import static org.springframework.test.web.client.match.MockRestRequestMatchers.requestTo; import static org.springframework.test.web.client.response.MockRestResponseCreators.withStatus; @@ -78,13 +76,11 @@ import static uk.ac.ebi.eva.accession.clustering.test.configuration.BatchTestConfiguration.JOB_LAUNCHER_RS_ACCESSION_RECOVERY; import static uk.ac.ebi.eva.accession.clustering.test.configuration.BatchTestConfiguration.JOB_LAUNCHER_STUDY_FROM_MONGO; -@RunWith(SpringRunner.class) -@ContextConfiguration(classes = {BatchTestConfiguration.class}) +@ExtendWith(SpringExtension.class) +@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class, + BatchJobRepositoryTestConfiguration.class}) @TestPropertySource("classpath:clustering-writer-test.properties") -public class ClusteringVariantJobConfigurationTest { - - private static final String TEST_DB = "test-db"; - +public class ClusteringVariantJobConfigurationTest extends MongoTestContainerHelper { @Autowired @Qualifier(JOB_LAUNCHER_FROM_MONGO) private JobLauncherTestUtils jobLauncherTestUtilsFromMongo; @@ -100,9 +96,8 @@ public class ClusteringVariantJobConfigurationTest { @Autowired private MongoTemplate mongoTemplate; - //Required by nosql-unit @Autowired - private ApplicationContext applicationContext; + private ResourceLoader resourceLoader; @Autowired private InputParameters inputParameters; @@ -118,11 +113,7 @@ public class ClusteringVariantJobConfigurationTest { private final String URL_PATH_SAVE_COUNT = "/v1/bulk/count"; - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName(TEST_DB).build()); - - @Before + @BeforeEach public void init() throws Exception { mongoTemplate.getDb().drop(); mockServer = MockRestServiceServer.createServer(restTemplate); @@ -131,15 +122,16 @@ public void init() throws Exception { .andRespond(withStatus(HttpStatus.OK)); } - @After + @AfterEach public void tearDown() { mongoTemplate.getDb().drop(); } @Test @DirtiesContext - @UsingDataSet(locations = {"/test-data/submittedVariantEntityMongoReader.json"}) public void jobFromMongo() throws Exception { + new MongoTestDataLoader(mongoTemplate, resourceLoader).load("/test-data/submittedVariantEntityMongoReader.json"); + JobExecution jobExecution = jobLauncherTestUtilsFromMongo.launchJob(); List expectedSteps = new ArrayList<>(); expectedSteps.add(CLUSTERING_CLUSTERED_VARIANTS_FROM_MONGO_STEP); @@ -156,8 +148,9 @@ public void jobFromMongo() throws Exception { @Test @DirtiesContext - @UsingDataSet(locations = {"/test-data/submittedVariantEntityStudyReader.json"}) public void studyJobFromMongo() throws Exception { + new MongoTestDataLoader(mongoTemplate, resourceLoader).load("/test-data/submittedVariantEntityStudyReader.json"); + JobExecution jobExecution = jobLauncherTestUtilsStudyFromMongo.launchJob(); List expectedSteps = new ArrayList<>(); expectedSteps.add(STUDY_CLUSTERING_STEP); @@ -168,8 +161,9 @@ public void studyJobFromMongo() throws Exception { @Test @DirtiesContext - @UsingDataSet(locations = {"/test-data/submittedVariantEntityStudyReader.json"}) public void testStudyJobFromMongoIdempotent() throws Exception { + new MongoTestDataLoader(mongoTemplate, resourceLoader).load("/test-data/submittedVariantEntityStudyReader.json"); + jobLauncherTestUtilsStudyFromMongo.launchJob(); DatabaseState databaseStateAfterFirstRun = DatabaseState.getCurrentDatabaseState(mongoTemplate); jobLauncherTestUtilsStudyFromMongo.launchJob(); @@ -192,17 +186,17 @@ public void assertDataThatExceedsChunkSizeIsFullyProcessed() throws Exception { createMergeCandidateEntriesThatExceedChunkSize(numOperations); JobExecution jobExecution = jobLauncherTestUtilsFromMongo.launchJob(); assertEquals(numOperations, - jobExecution.getStepExecutions() - .stream() - .filter(stepExecution -> stepExecution.getStepName() - .equals(PROCESS_RS_SPLIT_CANDIDATES_STEP)) - .findFirst().get().getReadCount()); + jobExecution.getStepExecutions() + .stream() + .filter(stepExecution -> stepExecution.getStepName() + .equals(PROCESS_RS_SPLIT_CANDIDATES_STEP)) + .findFirst().get().getReadCount()); assertEquals(numOperations, - jobExecution.getStepExecutions() - .stream() - .filter(stepExecution -> stepExecution.getStepName() - .equals(PROCESS_RS_MERGE_CANDIDATES_STEP)) - .findFirst().get().getReadCount()); + jobExecution.getStepExecutions() + .stream() + .filter(stepExecution -> stepExecution.getStepName() + .equals(PROCESS_RS_MERGE_CANDIDATES_STEP)) + .findFirst().get().getReadCount()); } @Test @@ -224,11 +218,11 @@ private void createMergeCandidateEntriesThatExceedChunkSize(int numSplitCandidat for (int i = 0; i < numSplitCandidateOperations; i++) { SubmittedVariantOperationEntity mergeOperation = new SubmittedVariantOperationEntity(); mergeOperation.fill(RSMergeAndSplitCandidatesReaderConfiguration.MERGE_CANDIDATES_EVENT_TYPE, - ss1.getAccession(), null, "Mock merge candidate", - Arrays.asList(ss1, ss2)); + ss1.getAccession(), null, "Mock merge candidate", + Arrays.asList(ss1, ss2)); mergeOperation.setId(MERGE_CANDIDATE_ID_PREFIX + "_" + ss1.getReferenceSequenceAccession() + "_" + i); mongoTemplate.insert(mergeOperation, - mongoTemplate.getCollectionName(SubmittedVariantOperationEntity.class)); + mongoTemplate.getCollectionName(SubmittedVariantOperationEntity.class)); } } @@ -241,20 +235,20 @@ private void createSplitCandidateEntriesThatExceedChunkSize(int numSplitCandidat for (int i = 0; i < numSplitCandidateOperations; i++) { SubmittedVariantOperationEntity splitOperation = new SubmittedVariantOperationEntity(); splitOperation.fill(RSMergeAndSplitCandidatesReaderConfiguration.SPLIT_CANDIDATES_EVENT_TYPE, - ss3.getAccession(), null, "Mock split candidate", - Arrays.asList(ss3, ss4)); + ss3.getAccession(), null, "Mock split candidate", + Arrays.asList(ss3, ss4)); splitOperation.setId(SPLIT_CANDIDATE_ID_PREFIX + "_" + ss3.getReferenceSequenceAccession() + "_" + i); mongoTemplate.insert(splitOperation, - mongoTemplate.getCollectionName(SubmittedVariantOperationEntity.class)); + mongoTemplate.getCollectionName(SubmittedVariantOperationEntity.class)); } } private SubmittedVariantEntity createSSWithLocus(Long ssAccession, Long rsAccession, Long start, String reference, String alternate) { return new SubmittedVariantEntity(ssAccession, "hash" + ssAccession, inputParameters.getAssemblyAccession(), - 60711, - "PRJ1", "chr1", start, reference, alternate, rsAccession, false, false, false, - false, 1); + 60711, + "PRJ1", "chr1", start, reference, alternate, rsAccession, false, false, false, + false, 1); } } \ No newline at end of file diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/RSAccessionRecoveryTest.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/RSAccessionRecoveryTest.java index dfbbfa08f..7df9926c5 100644 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/RSAccessionRecoveryTest.java +++ b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/RSAccessionRecoveryTest.java @@ -16,45 +16,64 @@ * limitations under the License. */ -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.batch.core.BatchStatus; import org.springframework.batch.core.JobExecution; import org.springframework.batch.test.JobLauncherTestUtils; import org.springframework.batch.test.context.SpringBatchTest; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.ampt2d.commons.accession.persistence.jpa.monotonic.entities.ContiguousIdBlock; import uk.ac.ebi.ampt2d.commons.accession.persistence.jpa.monotonic.repositories.ContiguousIdBlockRepository; +import uk.ac.ebi.eva.accession.clustering.test.configuration.BatchJobRepositoryTestConfiguration; +import uk.ac.ebi.eva.accession.clustering.test.configuration.MongoTestConfiguration; import uk.ac.ebi.eva.accession.clustering.test.configuration.RSAccessionRecoveryTestConfiguration; import uk.ac.ebi.eva.accession.core.model.ClusteredVariant; import uk.ac.ebi.eva.accession.core.model.eva.ClusteredVariantEntity; import uk.ac.ebi.eva.accession.core.repository.nonhuman.eva.ClusteredVariantAccessioningRepository; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.commons.core.models.VariantType; import java.time.LocalDateTime; import java.util.ArrayList; import java.util.List; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import static uk.ac.ebi.eva.accession.clustering.test.configuration.RSAccessionRecoveryTestConfiguration.JOB_LAUNCHER_RS_ACCESSION_RECOVERY; -@RunWith(SpringRunner.class) -@ContextConfiguration(classes = {RSAccessionRecoveryTestConfiguration.class,}) +@ExtendWith(SpringExtension.class) +@ContextConfiguration(classes = {RSAccessionRecoveryTestConfiguration.class, MongoTestConfiguration.class, BatchJobRepositoryTestConfiguration.class}) @TestPropertySource("classpath:rs-accession-recovery.properties") @SpringBatchTest -public class RSAccessionRecoveryTest { +public class RSAccessionRecoveryTest extends MongoTestContainerHelper { @Autowired private ContiguousIdBlockRepository blockRepository; @Autowired private ClusteredVariantAccessioningRepository mongoRepository; + @Autowired + private MongoTemplate mongoTemplate; + + @BeforeEach + public void setUp() { + mongoTemplate.getDb().drop(); + } + + @AfterEach + public void tearDown() { + mongoTemplate.getDb().drop(); + } + @Autowired @Qualifier(JOB_LAUNCHER_RS_ACCESSION_RECOVERY) private JobLauncherTestUtils jobLauncherTestUtilsMonotonicAccessionRecoveryAgent; diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/qc/DuplicateRSAccQCStepConfigurationTest.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/qc/DuplicateRSAccQCStepConfigurationTest.java index 98d2abc28..d3b308691 100644 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/qc/DuplicateRSAccQCStepConfigurationTest.java +++ b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/qc/DuplicateRSAccQCStepConfigurationTest.java @@ -15,32 +15,30 @@ */ package uk.ac.ebi.eva.accession.clustering.configuration.batch.jobs.qc; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; -import com.mongodb.MongoClient; -import gherkin.deps.com.google.gson.Gson; -import gherkin.deps.com.google.gson.GsonBuilder; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; import org.apache.commons.lang3.tuple.Pair; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.batch.core.BatchStatus; import org.springframework.batch.core.JobExecution; import org.springframework.batch.test.JobLauncherTestUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.context.ApplicationContext; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.ampt2d.commons.accession.hashing.SHA1HashingFunction; import uk.ac.ebi.eva.accession.clustering.batch.io.qc.DuplicateRSAccQCResult; +import uk.ac.ebi.eva.accession.clustering.test.configuration.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.accession.clustering.test.configuration.BatchTestConfiguration; import uk.ac.ebi.eva.accession.clustering.test.configuration.MongoTestConfiguration; -import uk.ac.ebi.eva.accession.clustering.test.rule.FixSpringMongoDbRule; import uk.ac.ebi.eva.accession.core.model.ClusteredVariant; import uk.ac.ebi.eva.accession.core.model.IClusteredVariant; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpClusteredVariantEntity; @@ -48,6 +46,7 @@ import uk.ac.ebi.eva.accession.core.model.eva.ClusteredVariantEntity; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; import uk.ac.ebi.eva.accession.core.summary.ClusteredVariantSummaryFunction; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.commons.core.models.pipeline.Variant; import java.io.IOException; @@ -61,48 +60,35 @@ import java.util.function.Function; import java.util.stream.Collectors; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.DUPLICATE_RS_ACC_QC_STEP; import static uk.ac.ebi.eva.accession.clustering.test.configuration.BatchTestConfiguration.JOB_LAUNCHER_DUPLICATE_RS_ACC_QC_JOB; -@RunWith(SpringRunner.class) -@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class}) +@ExtendWith(SpringExtension.class) +@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class, BatchJobRepositoryTestConfiguration.class}) @TestPropertySource("classpath:duplicate-rs-acc-qc-test.properties") -public class DuplicateRSAccQCStepConfigurationTest { - - private static final String TEST_DB = "test-db"; +public class DuplicateRSAccQCStepConfigurationTest extends MongoTestContainerHelper { private static final String duplicateRsAccFile = "src/test/resources/duplicateRSAcc.csv"; @Autowired @Qualifier(JOB_LAUNCHER_DUPLICATE_RS_ACC_QC_JOB) private JobLauncherTestUtils jobLauncherTestUtils; - @Autowired - private MongoClient mongoClient; - @Autowired private MongoTemplate mongoTemplate; - //Required by nosql-unit - @Autowired - private ApplicationContext applicationContext; - private final Gson gson = new GsonBuilder().create(); - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName(TEST_DB).build()); - - @Before + @BeforeEach public void setUp() throws IOException { - this.mongoClient.dropDatabase(TEST_DB); + mongoTemplate.getDb().drop(); Files.deleteIfExists(Paths.get(duplicateRsAccFile)); } - @After + @AfterEach public void tearDown() throws IOException { - this.mongoClient.dropDatabase(TEST_DB); + mongoTemplate.getDb().drop(); Files.deleteIfExists(Paths.get(duplicateRsAccFile)); } @@ -303,7 +289,31 @@ public void assertDuplicateRSAccFileContains(Set duplicateCVEAccessions, String expectedJsonData = gson.toJson(new DuplicateRSAccQCResult(cveAcc, mapOfCveAccAndCveEntities.get(cveAcc), mapOfSveGroupByCveAccAndThenByAsmLoc.get(cveAcc))); - assertEquals(expectedJsonData, fileJsonData); + assertEquals(normalizeNano(expectedJsonData), normalizeNano(fileJsonData)); + } + } + + private String normalizeNano(String json) { + JsonElement element = JsonParser.parseString(json); + removeNano(element); + return element.toString(); + } + + private void removeNano(JsonElement element) { + if (element.isJsonObject()) { + JsonObject obj = element.getAsJsonObject(); + + if (obj.has("nano")) { + obj.addProperty("nano", 0); + } + + for (Map.Entry entry : obj.entrySet()) { + removeNano(entry.getValue()); + } + } else if (element.isJsonArray()) { + for (JsonElement child : element.getAsJsonArray()) { + removeNano(child); + } } } diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/qc/NewClusteredVariantsQCJobConfigurationTest.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/qc/NewClusteredVariantsQCJobConfigurationTest.java index b50197932..91b020eb3 100644 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/qc/NewClusteredVariantsQCJobConfigurationTest.java +++ b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/qc/NewClusteredVariantsQCJobConfigurationTest.java @@ -15,54 +15,61 @@ */ package uk.ac.ebi.eva.accession.clustering.configuration.batch.jobs.qc; -import com.lordofthejars.nosqlunit.annotation.UsingDataSet; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.batch.core.BatchStatus; import org.springframework.batch.core.JobExecution; import org.springframework.batch.core.StepExecution; import org.springframework.batch.test.JobLauncherTestUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.context.ApplicationContext; +import org.springframework.core.io.ResourceLoader; +import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; - +import org.springframework.test.context.junit.jupiter.SpringExtension; +import uk.ac.ebi.eva.accession.clustering.test.configuration.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.accession.clustering.test.configuration.BatchTestConfiguration; -import uk.ac.ebi.eva.accession.clustering.test.rule.FixSpringMongoDbRule; +import uk.ac.ebi.eva.accession.clustering.test.configuration.MongoTestConfiguration; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; +import uk.ac.ebi.eva.accession.core.utils.MongoTestDataLoader; import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.stream.Collectors; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import static uk.ac.ebi.eva.accession.clustering.configuration.batch.jobs.qc.NewClusteredVariantsQCJobConfiguration.REPORT_MISSING_CVE_STEP; import static uk.ac.ebi.eva.accession.clustering.test.configuration.BatchTestConfiguration.JOB_LAUNCHER_NEW_CLUSTERED_VARIANTS_QC; -@RunWith(SpringRunner.class) -@ContextConfiguration(classes = {BatchTestConfiguration.class}) +@ExtendWith(SpringExtension.class) +@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class, BatchJobRepositoryTestConfiguration.class}) @TestPropertySource("classpath:clustering-qc-test.properties") -@UsingDataSet(locations = {"/test-data/clusteredVariantEntityQC.json"}) -public class NewClusteredVariantsQCJobConfigurationTest { - - private static final String TEST_DB = "test-db"; +public class NewClusteredVariantsQCJobConfigurationTest extends MongoTestContainerHelper { @Autowired - @Qualifier(JOB_LAUNCHER_NEW_CLUSTERED_VARIANTS_QC) - private JobLauncherTestUtils jobLauncherTestUtilsNewClusteredVariantsQC; + private MongoTemplate mongoTemplate; - //Required by nosql-unit @Autowired - private ApplicationContext applicationContext; + private ResourceLoader resourceLoader; + + @BeforeEach + public void setUp() { + mongoTemplate.getDb().drop(); + new MongoTestDataLoader(mongoTemplate, resourceLoader).load("/test-data/clusteredVariantEntityQC.json"); + } - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName(TEST_DB).build()); + @AfterEach + public void tearDown() { + mongoTemplate.getDb().drop(); + } + + @Autowired + @Qualifier(JOB_LAUNCHER_NEW_CLUSTERED_VARIANTS_QC) + private JobLauncherTestUtils jobLauncherTestUtilsNewClusteredVariantsQC; @Test public void newClusteredVariantsQCJob() throws Exception { diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/steps/ClusteringVariantStepConfigurationTest.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/steps/ClusteringVariantStepConfigurationTest.java index 28d17ad53..2aa9ddbe5 100644 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/steps/ClusteringVariantStepConfigurationTest.java +++ b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/steps/ClusteringVariantStepConfigurationTest.java @@ -15,38 +15,37 @@ */ package uk.ac.ebi.eva.accession.clustering.configuration.batch.steps; -import com.lordofthejars.nosqlunit.annotation.UsingDataSet; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; import com.mongodb.client.FindIterable; import com.mongodb.client.MongoCollection; import org.bson.Document; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.batch.core.BatchStatus; import org.springframework.batch.core.JobExecution; import org.springframework.batch.core.StepExecution; import org.springframework.batch.test.JobLauncherTestUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.context.ApplicationContext; +import org.springframework.core.io.ResourceLoader; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.http.HttpMethod; import org.springframework.http.HttpStatus; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import org.springframework.test.web.client.ExpectedCount; import org.springframework.test.web.client.MockRestServiceServer; import org.springframework.web.client.RestTemplate; import uk.ac.ebi.ampt2d.commons.accession.core.models.EventType; +import uk.ac.ebi.eva.accession.clustering.test.configuration.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.accession.clustering.test.configuration.BatchTestConfiguration; -import uk.ac.ebi.eva.accession.clustering.test.rule.FixSpringMongoDbRule; +import uk.ac.ebi.eva.accession.clustering.test.configuration.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantOperationEntity; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; +import uk.ac.ebi.eva.accession.core.utils.MongoTestDataLoader; import uk.ac.ebi.eva.metrics.count.CountServiceParameters; import java.net.URI; @@ -55,8 +54,8 @@ import java.util.Collections; import java.util.List; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.springframework.test.web.client.match.MockRestRequestMatchers.method; import static org.springframework.test.web.client.match.MockRestRequestMatchers.requestTo; import static org.springframework.test.web.client.response.MockRestResponseCreators.withStatus; @@ -67,14 +66,11 @@ import static uk.ac.ebi.eva.accession.clustering.test.configuration.BatchTestConfiguration.JOB_LAUNCHER_FROM_MONGO_ONLY_FIRST_STEP; import static uk.ac.ebi.eva.accession.clustering.test.configuration.BatchTestConfiguration.JOB_LAUNCHER_STUDY_FROM_MONGO; -@RunWith(SpringRunner.class) -@ContextConfiguration(classes = {BatchTestConfiguration.class}) +@ExtendWith(SpringExtension.class) +@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class, + BatchJobRepositoryTestConfiguration.class}) @TestPropertySource("classpath:clustering-issuance-test.properties") -@UsingDataSet(locations = {"/test-data/submittedVariantEntity.json"}) -public class ClusteringVariantStepConfigurationTest { - - private static final String TEST_DB = "test-db"; - +public class ClusteringVariantStepConfigurationTest extends MongoTestContainerHelper { private static final String CLUSTERED_VARIANT_COLLECTION = "clusteredVariantEntity"; private static final String SUBMITTED_VARIANT_COLLECTION = "submittedVariantEntity"; @@ -94,6 +90,9 @@ public class ClusteringVariantStepConfigurationTest { @Autowired private MongoTemplate mongoTemplate; + @Autowired + private ResourceLoader resourceLoader; + private MockRestServiceServer mockServer; @Autowired @@ -105,23 +104,17 @@ public class ClusteringVariantStepConfigurationTest { private final String URL_PATH_SAVE_COUNT = "/v1/bulk/count"; - //Required by nosql-unit - @Autowired - private ApplicationContext applicationContext; - - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName(TEST_DB).build()); - - @Before + @BeforeEach public void init() throws Exception { + mongoTemplate.getDb().drop(); + mockServer = MockRestServiceServer.createServer(restTemplate); mockServer.expect(ExpectedCount.manyTimes(), requestTo(new URI(countServiceParameters.getUrl() + URL_PATH_SAVE_COUNT))) .andExpect(method(HttpMethod.POST)) .andRespond(withStatus(HttpStatus.OK)); } - @After + @AfterEach public void tearDown() { mongoTemplate.getDb().drop(); } @@ -129,6 +122,8 @@ public void tearDown() { @Test @DirtiesContext public void nonClusteredVariantStepFromMongo() { + new MongoTestDataLoader(mongoTemplate, resourceLoader).load("/test-data/submittedVariantEntity.json"); + assertEquals(5, mongoTemplate.getCollection(SUBMITTED_VARIANT_COLLECTION).countDocuments()); assertTrue(allSubmittedVariantsNotClustered()); @@ -143,8 +138,9 @@ public void nonClusteredVariantStepFromMongo() { @Test @DirtiesContext - @UsingDataSet(locations = {"/test-data/submittedVariantEntityStudyReader.json"}) public void clusteredVariantStepStudyFromMongo() { + new MongoTestDataLoader(mongoTemplate, resourceLoader).load("/test-data/submittedVariantEntityStudyReader.json"); + assertEquals(5, mongoTemplate.getCollection(SUBMITTED_VARIANT_COLLECTION).countDocuments()); assertEquals(2, getSubmittedVariantsWithRSOrBackPropRS()); @@ -158,9 +154,11 @@ public void clusteredVariantStepStudyFromMongo() { @Test @DirtiesContext - @UsingDataSet(locations = {"/test-data/submittedVariantEntityMongoReader.json", - "/test-data/clusteredVariantEntityMongoReader.json"}) public void clusteredVariantStepFromMongo() throws Exception { + MongoTestDataLoader mongoTestDataLoader = new MongoTestDataLoader(mongoTemplate, resourceLoader); + mongoTestDataLoader.load("/test-data/submittedVariantEntityMongoReader.json"); + mongoTestDataLoader.load("/test-data/clusteredVariantEntityMongoReader.json"); + assertEquals(6, mongoTemplate.getCollection(SUBMITTED_VARIANT_COLLECTION).countDocuments()); assertEquals(1, getSubmittedVariantsWithRSOrBackPropRS()); @@ -168,7 +166,7 @@ public void clusteredVariantStepFromMongo() throws Exception { // the following cannot be executed directly, therefore we launch the entire job and ensure that the // CLUSTERING_CLUSTERED_VARIANTS_FROM_MONGO_STEP step was completed //JobExecution jobExecution = jobLauncherTestUtilsFromMongo.launchStep( - //CLUSTERING_CLUSTERED_VARIANTS_FROM_MONGO_STEP); + //CLUSTERING_CLUSTERED_VARIANTS_FROM_MONGO_STEP); JobExecution jobExecution = jobLauncherTestUtilsFromMongoOnlyFirstStep.launchJob(); assertEquals(BatchStatus.COMPLETED, jobExecution.getStatus()); assertTrue(jobExecution.getStepExecutions().stream().map(StepExecution::getStepName) diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/runner/ClusteringCommandLineRunnerTest.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/runner/ClusteringCommandLineRunnerTest.java index dfdd4ddba..7206b6cf1 100644 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/runner/ClusteringCommandLineRunnerTest.java +++ b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/runner/ClusteringCommandLineRunnerTest.java @@ -16,22 +16,19 @@ */ package uk.ac.ebi.eva.accession.clustering.runner; -import com.lordofthejars.nosqlunit.annotation.UsingDataSet; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mockito; import org.springframework.batch.core.JobExecution; import org.springframework.batch.core.JobExecutionException; import org.springframework.batch.core.JobInstance; import org.springframework.batch.core.explore.JobExplorer; import org.springframework.batch.core.repository.JobRepository; +import org.springframework.batch.item.Chunk; import org.springframework.batch.item.ExecutionContext; import org.springframework.batch.item.ItemWriter; import org.springframework.batch.test.JobLauncherTestUtils; @@ -39,7 +36,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.test.mock.mockito.MockBean; -import org.springframework.context.ApplicationContext; +import org.springframework.core.io.ResourceLoader; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.data.mongodb.core.query.Query; import org.springframework.http.HttpMethod; @@ -47,7 +44,7 @@ import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import org.springframework.test.web.client.ExpectedCount; import org.springframework.test.web.client.MockRestServiceServer; import org.springframework.web.client.RestTemplate; @@ -65,8 +62,9 @@ import uk.ac.ebi.eva.accession.clustering.configuration.batch.io.RSMergeAndSplitCandidatesReaderConfiguration; import uk.ac.ebi.eva.accession.clustering.parameters.InputParameters; import uk.ac.ebi.eva.accession.clustering.test.DatabaseState; +import uk.ac.ebi.eva.accession.clustering.test.configuration.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.accession.clustering.test.configuration.BatchTestConfiguration; -import uk.ac.ebi.eva.accession.clustering.test.rule.FixSpringMongoDbRule; +import uk.ac.ebi.eva.accession.clustering.test.configuration.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.model.ClusteredVariant; import uk.ac.ebi.eva.accession.core.model.IClusteredVariant; import uk.ac.ebi.eva.accession.core.model.ISubmittedVariant; @@ -84,12 +82,13 @@ import uk.ac.ebi.eva.accession.core.service.nonhuman.SubmittedVariantAccessioningService; import uk.ac.ebi.eva.accession.core.summary.ClusteredVariantSummaryFunction; import uk.ac.ebi.eva.accession.core.summary.SubmittedVariantSummaryFunction; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; +import uk.ac.ebi.eva.accession.core.utils.MongoTestDataLoader; import uk.ac.ebi.eva.commons.core.models.VariantClassifier; import uk.ac.ebi.eva.commons.core.models.VariantType; import uk.ac.ebi.eva.commons.mongodb.readers.MongoDbCursorItemReader; import uk.ac.ebi.eva.metrics.count.CountServiceParameters; -import javax.sql.DataSource; import java.io.File; import java.net.URI; import java.util.ArrayList; @@ -101,26 +100,36 @@ import java.util.function.Function; import java.util.stream.Collectors; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.springframework.data.mongodb.core.query.Criteria.where; import static org.springframework.data.mongodb.core.query.Query.query; import static org.springframework.test.web.client.match.MockRestRequestMatchers.method; import static org.springframework.test.web.client.match.MockRestRequestMatchers.requestTo; import static org.springframework.test.web.client.response.MockRestResponseCreators.withStatus; -import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.*; -import static uk.ac.ebi.eva.accession.clustering.test.configuration.BatchTestConfiguration.JOB_LAUNCHER_FROM_MONGO; +import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.BACK_PROPAGATE_NEW_RS_STEP; +import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.BACK_PROPAGATE_SPLIT_OR_MERGED_RS_STEP; +import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.CLEAR_RS_MERGE_AND_SPLIT_CANDIDATES_STEP; +import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.CLUSTERED_CLUSTERING_WRITER; +import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.CLUSTERING_CLUSTERED_VARIANTS_FROM_MONGO_STEP; +import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.CLUSTERING_FROM_MONGO_JOB; +import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.CLUSTERING_NON_CLUSTERED_VARIANTS_FROM_MONGO_STEP; +import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.CLUSTER_UNCLUSTERED_VARIANTS_JOB; +import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.PROCESS_REMAPPED_VARIANTS_WITH_RS_JOB; +import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.PROCESS_RS_MERGE_CANDIDATES_STEP; +import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.PROCESS_RS_SPLIT_CANDIDATES_STEP; +import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.RS_MERGE_WRITER; +import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.RS_SPLIT_WRITER; import static uk.ac.ebi.eva.accession.clustering.test.configuration.BatchTestConfiguration.JOB_LAUNCHER_FROM_MONGO_ONLY_FIRST_STEP; -@RunWith(SpringRunner.class) -@ContextConfiguration(classes = {BatchTestConfiguration.class}) +@ExtendWith(SpringExtension.class) +@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class, + BatchJobRepositoryTestConfiguration.class}) @TestPropertySource("classpath:clustering-pipeline-test.properties") -public class ClusteringCommandLineRunnerTest { +public class ClusteringCommandLineRunnerTest extends MongoTestContainerHelper { private static String TEST_APPLICATION_INSTANCE_ID = "test-application-instance-id"; - private static final String TEST_DB = "test-db"; - private static final int TAXONOMY = 60711; private static String ASM1; @@ -156,18 +165,14 @@ public class ClusteringCommandLineRunnerTest { @Autowired private JobExplorer jobExplorer; - @Autowired - private DataSource datasource; - @Autowired private ClusteringCommandLineRunner runner; @Autowired private MongoTemplate mongoTemplate; - //Required by nosql-unit @Autowired - private ApplicationContext applicationContext; + private ResourceLoader resourceLoader; @Autowired private MonotonicAccessionGenerator clusteredVariantAccessionGenerator; @@ -182,25 +187,10 @@ public class ClusteringCommandLineRunnerTest { @Qualifier(CLUSTERED_CLUSTERING_WRITER) private ClusteringWriter clusteringWriter; - @Autowired - @Qualifier(JOB_LAUNCHER_FROM_MONGO) - private JobLauncherTestUtils jobLauncherTestUtilsFromMongo; - @Autowired @Qualifier(JOB_LAUNCHER_FROM_MONGO_ONLY_FIRST_STEP) private JobLauncherTestUtils jobLauncherTestUtilsFromMongoOnlyFirstStep; - // Current clustering sequence is: - // generate merge split candidates from clustered variants -> perform merge - // -> perform split -> cluster new variants - @Autowired - @Qualifier(CLUSTERED_CLUSTERING_WRITER) - private ClusteringWriter clusteringWriterPreMergeAndSplit; - - @Autowired - @Qualifier(NON_CLUSTERED_CLUSTERING_WRITER) - private ClusteringWriter clusteringWriterPostMergeAndSplit; - private MongoDbCursorItemReader rsMergeCandidatesReader; private MongoDbCursorItemReader rsSplitCandidatesReader; @@ -216,11 +206,6 @@ public class ClusteringCommandLineRunnerTest { @MockBean private JobExecution jobExecution; - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName(TEST_DB).build()); - - private JobRepositoryTestUtils jobRepositoryTestUtils; private static String originalRemappedFrom; @@ -269,17 +254,17 @@ public String getHash() { private final String URL_PATH_SAVE_COUNT = "/v1/bulk/count"; - @BeforeClass + @BeforeAll public static void initializeTempFile() throws Exception { tempVcfInputFileToTestFailingJobs = File.createTempFile("resumeFailingJob", ".vcf.gz"); } - @AfterClass - public static void deleteTempFile() throws Exception { + @AfterAll + public static void deleteTempFile() { tempVcfInputFileToTestFailingJobs.delete(); } - @Before + @BeforeEach public void setUp() throws Exception { if (!originalInputParametersCaptured) { originalRemappedFrom = inputParameters.getRemappedFrom(); @@ -287,7 +272,7 @@ public void setUp() throws Exception { } ASM1 = inputParameters.getRemappedFrom(); ASM2 = inputParameters.getAssemblyAccession(); - jobRepositoryTestUtils = new JobRepositoryTestUtils(jobRepository, datasource); + jobRepositoryTestUtils = new JobRepositoryTestUtils(jobRepository); jobRepositoryTestUtils.removeJobExecutions(); inputParameters.setForceRestart(false); inputParameters.setRemappedFrom(originalRemappedFrom); @@ -302,7 +287,7 @@ public void setUp() throws Exception { rsSplitWriter.setJobExecution(jobExecution); } - @After + @AfterEach public void tearDown() { jobRepositoryTestUtils.removeJobExecutions(); inputParameters.setForceRestart(false); @@ -310,10 +295,11 @@ public void tearDown() { } @Test - @UsingDataSet(locations = {"/test-data/submittedVariantEntityMongoReader.json"}) @DirtiesContext public void runFullClusteringFromMongoJobWithNoErrors() throws JobExecutionException { - runner.setJobNames(CLUSTERING_FROM_MONGO_JOB); + new MongoTestDataLoader(mongoTemplate, resourceLoader).load("/test-data/submittedVariantEntityMongoReader.json"); + + runner.setJobName(CLUSTERING_FROM_MONGO_JOB); runner.run(); assertEquals(ClusteringCommandLineRunner.EXIT_WITHOUT_ERRORS, runner.getExitCode()); @@ -338,13 +324,14 @@ public void runFullClusteringFromMongoJobWithNoErrors() throws JobExecutionExcep } @Test - @UsingDataSet(locations = {"/test-data/submittedVariantEntityMongoReader.json"}) @DirtiesContext // For a clustering job involving non-remapped variants, only one step i.e., // CLUSTERING_NON_CLUSTERED_VARIANTS_FROM_MONGO_STEP should be executed public void runPartialClusteringFromMongoJobForNonRemappedVariants() throws JobExecutionException { + new MongoTestDataLoader(mongoTemplate, resourceLoader).load("/test-data/submittedVariantEntityMongoReader.json"); + inputParameters.setRemappedFrom(null); - runner.setJobNames(CLUSTERING_FROM_MONGO_JOB); + runner.setJobName(CLUSTERING_FROM_MONGO_JOB); runner.run(); assertEquals(ClusteringCommandLineRunner.EXIT_WITHOUT_ERRORS, runner.getExitCode()); @@ -368,10 +355,11 @@ public void runPartialClusteringFromMongoJobForNonRemappedVariants() throws JobE } @Test - @UsingDataSet(locations = {"/test-data/submittedVariantEntityMongoReader.json"}) @DirtiesContext public void runProcessRemappedRSJobWithNoErrors() throws JobExecutionException { - runner.setJobNames(PROCESS_REMAPPED_VARIANTS_WITH_RS_JOB); + new MongoTestDataLoader(mongoTemplate, resourceLoader).load("/test-data/submittedVariantEntityMongoReader.json"); + + runner.setJobName(PROCESS_REMAPPED_VARIANTS_WITH_RS_JOB); runner.run(); assertEquals(ClusteringCommandLineRunner.EXIT_WITHOUT_ERRORS, runner.getExitCode()); } @@ -395,7 +383,7 @@ public void runProcessRemappedRSJobMultipleTimes() throws JobExecutionException SubmittedVariantEntity ss1 = createSS(1L, 1L, rsLocus1, "A", "T", true); SubmittedVariantEntity ss2 = createSS(2L, 2L, rsLocus1, "A", "G", true); SubmittedVariantEntity ss3 = createSS(3L, 3L, rsLocus2, "T", "", true); - runner.setJobNames(PROCESS_REMAPPED_VARIANTS_WITH_RS_JOB); + runner.setJobName(PROCESS_REMAPPED_VARIANTS_WITH_RS_JOB); runner.run(); assertEquals(ClusteringCommandLineRunner.EXIT_WITHOUT_ERRORS, runner.getExitCode()); mergeOperations.add(Arrays.asList(ss1, ss2)); @@ -405,7 +393,7 @@ public void runProcessRemappedRSJobMultipleTimes() throws JobExecutionException // Second run SubmittedVariantEntity ss4 = createSS(4L, 3L, rsLocus3, "", "T", true); SubmittedVariantEntity ss5 = createSS(5L, 4L, rsLocus4, "A", "G", true); - runner.setJobNames(PROCESS_REMAPPED_VARIANTS_WITH_RS_JOB); + runner.setJobName(PROCESS_REMAPPED_VARIANTS_WITH_RS_JOB); runner.run(); assertEquals(ClusteringCommandLineRunner.EXIT_WITHOUT_ERRORS, runner.getExitCode()); splitOperations.add(Arrays.asList(ss3, ss4)); @@ -414,7 +402,7 @@ public void runProcessRemappedRSJobMultipleTimes() throws JobExecutionException // Third run SubmittedVariantEntity ss6 = createSS(6L, 5L, rsLocus4, "G", "A", true); - runner.setJobNames(PROCESS_REMAPPED_VARIANTS_WITH_RS_JOB); + runner.setJobName(PROCESS_REMAPPED_VARIANTS_WITH_RS_JOB); runner.run(); assertEquals(ClusteringCommandLineRunner.EXIT_WITHOUT_ERRORS, runner.getExitCode()); mergeOperations.add(Arrays.asList(ss5, ss6)); @@ -422,7 +410,7 @@ public void runProcessRemappedRSJobMultipleTimes() throws JobExecutionException // Fourth run createSS(7L, 6L, rsLocus5, "", "A", true); - runner.setJobNames(PROCESS_REMAPPED_VARIANTS_WITH_RS_JOB); + runner.setJobName(PROCESS_REMAPPED_VARIANTS_WITH_RS_JOB); runner.run(); assertEquals(ClusteringCommandLineRunner.EXIT_WITHOUT_ERRORS, runner.getExitCode()); insertedRSHashes.add(rsLocus5.getHash()); @@ -495,10 +483,11 @@ private boolean areAllSSPresentInOperation(List expected } @Test - @UsingDataSet(locations = {"/test-data/submittedVariantEntityMongoReader.json"}) @DirtiesContext public void runClusterUnclusteredVariantsJobWithNoErrors() throws JobExecutionException { - runner.setJobNames(CLUSTER_UNCLUSTERED_VARIANTS_JOB); + new MongoTestDataLoader(mongoTemplate, resourceLoader).load("/test-data/submittedVariantEntityMongoReader.json"); + + runner.setJobName(CLUSTER_UNCLUSTERED_VARIANTS_JOB); runner.run(); assertEquals(ClusteringCommandLineRunner.EXIT_WITHOUT_ERRORS, runner.getExitCode()); } @@ -511,7 +500,7 @@ public void runClusterUnclusteredVariantsJobWithNoErrors() throws JobExecutionEx public void runBackPropagateRSJobWithNoErrors() throws JobExecutionException, AccessionCouldNotBeGeneratedException { setupRSAndSSInNewAndRemappedAssemblies(); - runner.setJobNames(CLUSTERING_FROM_MONGO_JOB); + runner.setJobName(CLUSTERING_FROM_MONGO_JOB); runner.run(); assertEquals(ClusteringCommandLineRunner.EXIT_WITHOUT_ERRORS, runner.getExitCode()); @@ -549,7 +538,7 @@ public void runClusteringMongoJobWithOverlappingMergesAndSplits() throws JobExec SubmittedVariantEntity ss2_another = createSS(2L, rs3.getAccession(), rsLocus2, "A", "", true); SubmittedVariantEntity ss3 = createSS(3L, rs2.getAccession(), rsLocus2, "C", "", true); - runner.setJobNames(CLUSTERING_FROM_MONGO_JOB); + runner.setJobName(CLUSTERING_FROM_MONGO_JOB); runner.run(); assertEquals(ClusteringCommandLineRunner.EXIT_WITHOUT_ERRORS, runner.getExitCode()); @@ -577,7 +566,7 @@ Initial state and expected results here (marked in green): https://docs.google.c setupRSAndSS(); createEVASS8InASM1WithUnassignedRS(); - runner.setJobNames(CLUSTERING_FROM_MONGO_JOB); + runner.setJobName(CLUSTERING_FROM_MONGO_JOB); runner.run(); assertEquals(ClusteringCommandLineRunner.EXIT_WITHOUT_ERRORS, runner.getExitCode()); @@ -910,7 +899,7 @@ private ClusteredVariantEntity createRS(Long rsAccession, RSLocus rsLocus, boole @Test @DirtiesContext public void runJobWithNoErrors() throws JobExecutionException { - runner.setJobNames(CLUSTERING_FROM_MONGO_JOB); + runner.setJobName(CLUSTERING_FROM_MONGO_JOB); runner.run(); assertEquals(ClusteringCommandLineRunner.EXIT_WITHOUT_ERRORS, runner.getExitCode()); } @@ -918,7 +907,7 @@ public void runJobWithNoErrors() throws JobExecutionException { @Test @DirtiesContext public void runJobWithNoName() throws JobExecutionException { - runner.setJobNames(null); + runner.setJobName(null); runner.run(); assertEquals(ClusteringCommandLineRunner.EXIT_WITH_ERRORS, runner.getExitCode()); } @@ -926,7 +915,7 @@ public void runJobWithNoName() throws JobExecutionException { @Test @DirtiesContext public void runNonExistentJob() throws JobExecutionException { - runner.setJobNames("NOT_EXISTENT_JOB"); + runner.setJobName("NOT_EXISTENT_JOB"); runner.run(); assertEquals(ClusteringCommandLineRunner.EXIT_WITH_ERRORS, runner.getExitCode()); } @@ -934,7 +923,7 @@ public void runNonExistentJob() throws JobExecutionException { @Test @DirtiesContext public void restartCompletedJobThatIsAlreadyInTheRepository() throws Exception { - runner.setJobNames(CLUSTERING_FROM_MONGO_JOB); + runner.setJobName(CLUSTERING_FROM_MONGO_JOB); runner.run(); assertEquals(ClusteringCommandLineRunner.EXIT_WITHOUT_ERRORS, runner.getExitCode()); @@ -947,7 +936,7 @@ public void restartCompletedJobThatIsAlreadyInTheRepository() throws Exception { @Test @DirtiesContext public void forceRestartButNoJobInTheRepository() throws Exception { - runner.setJobNames(CLUSTERING_FROM_MONGO_JOB); + runner.setJobName(CLUSTERING_FROM_MONGO_JOB); inputParameters.setForceRestart(true); assertEquals(Collections.EMPTY_LIST, jobExplorer.getJobNames()); runner.run(); @@ -990,13 +979,13 @@ private void runSplitAndMergeIdentificationAndHandlingSteps() while ((tempSVO = rsMergeCandidatesReader.read()) != null) { mergeCandidates.add(tempSVO); } - rsMergeWriter.write(mergeCandidates); + rsMergeWriter.write(new Chunk<>(mergeCandidates)); rsSplitCandidatesReader = new RSMergeAndSplitCandidatesReaderConfiguration() .rsSplitCandidatesReader(this.mongoTemplate, this.inputParameters); rsSplitCandidatesReader.open(new ExecutionContext()); while ((tempSVO = rsSplitCandidatesReader.read()) != null) { splitCandidates.add(tempSVO); } - rsSplitWriter.write(splitCandidates); + rsSplitWriter.write(new Chunk<>(splitCandidates)); } } \ No newline at end of file diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/test/VariantAssertions.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/test/VariantAssertions.java index 5793cece4..e019219f0 100644 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/test/VariantAssertions.java +++ b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/test/VariantAssertions.java @@ -23,7 +23,7 @@ import java.util.TreeSet; import java.util.stream.Collectors; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; public class VariantAssertions { diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/test/configuration/BatchJobRepositoryTestConfiguration.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/test/configuration/BatchJobRepositoryTestConfiguration.java new file mode 100644 index 000000000..6cebe2481 --- /dev/null +++ b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/test/configuration/BatchJobRepositoryTestConfiguration.java @@ -0,0 +1,22 @@ +package uk.ac.ebi.eva.accession.clustering.test.configuration; + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Primary; +import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseBuilder; +import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseType; + +import javax.sql.DataSource; + +@Configuration +public class BatchJobRepositoryTestConfiguration { + + @Bean + @Primary + public DataSource dataSource() { + return new EmbeddedDatabaseBuilder() + .setType(EmbeddedDatabaseType.HSQL) + .addScript("org/springframework/batch/core/schema-hsqldb.sql") + .build(); + } +} \ No newline at end of file diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/test/configuration/BatchTestConfiguration.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/test/configuration/BatchTestConfiguration.java index 2fc8d7ed1..cc5eb69f8 100644 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/test/configuration/BatchTestConfiguration.java +++ b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/test/configuration/BatchTestConfiguration.java @@ -18,15 +18,15 @@ package uk.ac.ebi.eva.accession.clustering.test.configuration; import org.springframework.batch.core.Job; +import org.springframework.batch.core.launch.JobLauncher; +import org.springframework.batch.core.repository.JobRepository; import org.springframework.batch.test.JobLauncherTestUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; -import org.springframework.boot.autoconfigure.batch.BatchProperties; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Import; import org.springframework.core.io.ResourceLoader; -import org.springframework.transaction.PlatformTransactionManager; import uk.ac.ebi.eva.accession.clustering.configuration.batch.io.BackPropagatedRSWriterConfiguration; import uk.ac.ebi.eva.accession.clustering.configuration.batch.io.ClusteringMongoReaderConfiguration; import uk.ac.ebi.eva.accession.clustering.configuration.batch.io.ClusteringWriterConfiguration; @@ -57,8 +57,6 @@ import uk.ac.ebi.eva.accession.clustering.runner.ClusteringCommandLineRunner; import uk.ac.ebi.eva.commons.batch.job.JobExecutionApplicationListener; -import javax.sql.DataSource; - import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.CLUSTERING_FROM_MONGO_JOB; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.DUPLICATE_RS_ACC_QC_JOB; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.PROCESS_REMAPPED_VARIANTS_WITH_RS_JOB; @@ -110,76 +108,63 @@ public class BatchTestConfiguration { public static final String JOB_LAUNCHER_DUPLICATE_RS_ACC_QC_JOB = "JOB_LAUNCHER_DUPLICATE_RS_ACC_QC_JOB"; - @Autowired - private BatchProperties properties; - @Autowired private ResourceLoader resourceLoader; - @Autowired - private DataSource dataSource; - - @Autowired - private PlatformTransactionManager platformTransactionManager; - @Bean(JOB_LAUNCHER_FROM_MONGO) - public JobLauncherTestUtils jobLauncherTestUtilsFromMongo() { - - return new JobLauncherTestUtils() { - @Override - @Autowired - public void setJob(@Qualifier(CLUSTERING_FROM_MONGO_JOB) Job job) { - super.setJob(job); - } - }; + public JobLauncherTestUtils jobLauncherTestUtilsFromMongo(JobLauncher jobLauncher, JobRepository jobRepository, + @Qualifier(CLUSTERING_FROM_MONGO_JOB) Job job) { + JobLauncherTestUtils utils = new JobLauncherTestUtils(); + utils.setJobLauncher(jobLauncher); + utils.setJobRepository(jobRepository); + utils.setJob(job); + return utils; } @Bean(JOB_LAUNCHER_STUDY_FROM_MONGO) - public JobLauncherTestUtils jobLauncherTestUtilsStudyFromMongo() { - - return new JobLauncherTestUtils() { - @Override - @Autowired - public void setJob(@Qualifier(STUDY_CLUSTERING_JOB) Job job) { - super.setJob(job); - } - }; + public JobLauncherTestUtils jobLauncherTestUtilsStudyFromMongo(JobLauncher jobLauncher, JobRepository jobRepository, + @Qualifier(STUDY_CLUSTERING_JOB) Job job) { + + JobLauncherTestUtils utils = new JobLauncherTestUtils(); + utils.setJobLauncher(jobLauncher); + utils.setJobRepository(jobRepository); + utils.setJob(job); + return utils; } @Bean(JOB_LAUNCHER_RS_ACCESSION_RECOVERY) - public JobLauncherTestUtils jobLauncherTestUtilsRSAccessionRecoveryJob() { - - return new JobLauncherTestUtils() { - @Override - @Autowired - public void setJob(@Qualifier(RS_ACCESSION_RECOVERY_JOB) Job job) { - super.setJob(job); - } - }; + public JobLauncherTestUtils jobLauncherTestUtilsRSAccessionRecoveryJob(JobLauncher jobLauncher, JobRepository jobRepository, + @Qualifier(RS_ACCESSION_RECOVERY_JOB) Job job) { + + JobLauncherTestUtils utils = new JobLauncherTestUtils(); + utils.setJobLauncher(jobLauncher); + utils.setJobRepository(jobRepository); + utils.setJob(job); + return utils; } @Bean(JOB_LAUNCHER_NEW_CLUSTERED_VARIANTS_QC) - public JobLauncherTestUtils jobLauncherTestUtilsNewClusteredVariants() { - - return new JobLauncherTestUtils() { - @Override - @Autowired - public void setJob(@Qualifier(NEW_CLUSTERED_VARIANTS_QC_JOB) Job job) { - super.setJob(job); - } - }; + public JobLauncherTestUtils jobLauncherTestUtilsNewClusteredVariants(JobLauncher jobLauncher, JobRepository jobRepository, + @Qualifier(NEW_CLUSTERED_VARIANTS_QC_JOB) Job job) { + + JobLauncherTestUtils utils = new JobLauncherTestUtils(); + utils.setJobLauncher(jobLauncher); + utils.setJobRepository(jobRepository); + utils.setJob(job); + return utils; + } @Bean(JOB_LAUNCHER_FROM_MONGO_ONLY_FIRST_STEP) - public JobLauncherTestUtils jobLauncherTestUtilsFromMongoOnlyFirstStep() { - - return new JobLauncherTestUtils() { - @Override - @Autowired - public void setJob(@Qualifier(PROCESS_REMAPPED_VARIANTS_WITH_RS_JOB) Job job) { - super.setJob(job); - } - }; + public JobLauncherTestUtils jobLauncherTestUtilsFromMongoOnlyFirstStep(JobLauncher jobLauncher, JobRepository jobRepository, + @Qualifier(PROCESS_REMAPPED_VARIANTS_WITH_RS_JOB) Job job) { + + JobLauncherTestUtils utils = new JobLauncherTestUtils(); + utils.setJobLauncher(jobLauncher); + utils.setJobRepository(jobRepository); + utils.setJob(job); + return utils; + } @Bean @@ -188,13 +173,13 @@ public JobExecutionApplicationListener jobExecutionApplicationListener() { } @Bean(JOB_LAUNCHER_DUPLICATE_RS_ACC_QC_JOB) - public JobLauncherTestUtils jobLauncherTestUtilsDuplicateRSAccQCJob() { - return new JobLauncherTestUtils() { - @Override - @Autowired - public void setJob(@Qualifier(DUPLICATE_RS_ACC_QC_JOB) Job job) { - super.setJob(job); - } - }; + public JobLauncherTestUtils jobLauncherTestUtilsDuplicateRSAccQCJob(JobLauncher jobLauncher, JobRepository jobRepository, + @Qualifier(DUPLICATE_RS_ACC_QC_JOB) Job job) { + + JobLauncherTestUtils utils = new JobLauncherTestUtils(); + utils.setJobLauncher(jobLauncher); + utils.setJobRepository(jobRepository); + utils.setJob(job); + return utils; } } diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/test/configuration/MongoTestConfiguration.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/test/configuration/MongoTestConfiguration.java index 6bf441aca..a4a8fc2c0 100644 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/test/configuration/MongoTestConfiguration.java +++ b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/test/configuration/MongoTestConfiguration.java @@ -15,12 +15,30 @@ */ package uk.ac.ebi.eva.accession.clustering.test.configuration; +import com.mongodb.ReadPreference; +import com.mongodb.WriteConcern; +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoClients; +import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.domain.EntityScan; import org.springframework.boot.test.autoconfigure.data.mongo.AutoConfigureDataMongo; +import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Primary; +import org.springframework.data.mongodb.MongoDatabaseFactory; import org.springframework.data.mongodb.config.EnableMongoAuditing; +import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.data.mongodb.core.SimpleMongoClientDatabaseFactory; +import org.springframework.data.mongodb.core.WriteResultChecking; +import org.springframework.data.mongodb.core.convert.DefaultDbRefResolver; +import org.springframework.data.mongodb.core.convert.DefaultMongoTypeMapper; +import org.springframework.data.mongodb.core.convert.MappingMongoConverter; +import org.springframework.data.mongodb.core.convert.MongoCustomConversions; +import org.springframework.data.mongodb.core.mapping.MongoMappingContext; import org.springframework.data.mongodb.repository.config.EnableMongoRepositories; +import java.util.Collections; + @Configuration @EntityScan(basePackages = {"uk.ac.ebi.eva.accession.core.persistence"}) @EnableMongoRepositories(basePackages = { @@ -29,4 +47,56 @@ @EnableMongoAuditing @AutoConfigureDataMongo public class MongoTestConfiguration { -} + + @Value("${spring.data.mongodb.uri}") + private String mongoUri; + + @Value("${spring.data.mongodb.database}") + private String database; + + @Value("${mongodb.read-preference:primary}") + private String readPreference; + + @Bean + @Primary + public MongoClient mongoClient() { + return MongoClients.create(mongoUri); + } + + @Bean("primaryFactory") + @Primary + public MongoDatabaseFactory mongoDbFactory() { + return new SimpleMongoClientDatabaseFactory(mongoClient(), database); + } + + @Bean + @Primary + public MongoCustomConversions mongoCustomConversions() { + return new MongoCustomConversions(Collections.emptyList()); + } + + @Bean + @Primary + public MongoMappingContext mongoMappingContext(MongoCustomConversions conversions) { + MongoMappingContext context = new MongoMappingContext(); + context.setSimpleTypeHolder(conversions.getSimpleTypeHolder()); + context.setAutoIndexCreation(false); + return context; + } + + @Bean + @Primary + public MongoTemplate mongoTemplate(MongoMappingContext mongoMappingContext, + MongoCustomConversions conversions) { + MappingMongoConverter converter = new MappingMongoConverter( + new DefaultDbRefResolver(mongoDbFactory()), mongoMappingContext); + converter.setCustomConversions(conversions); + converter.setTypeMapper(new DefaultMongoTypeMapper(null)); + converter.afterPropertiesSet(); + MongoTemplate mongoTemplate = new MongoTemplate(mongoDbFactory(), converter); + mongoTemplate.setReadPreference(ReadPreference.valueOf(readPreference)); + mongoTemplate.setWriteConcern(WriteConcern.MAJORITY); + mongoTemplate.setWriteResultChecking(WriteResultChecking.EXCEPTION); + return mongoTemplate; + } +} \ No newline at end of file diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/test/configuration/RSAccessionRecoveryTestConfiguration.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/test/configuration/RSAccessionRecoveryTestConfiguration.java index b1bdcf1ab..6cd5d315c 100644 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/test/configuration/RSAccessionRecoveryTestConfiguration.java +++ b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/test/configuration/RSAccessionRecoveryTestConfiguration.java @@ -17,9 +17,9 @@ package uk.ac.ebi.eva.accession.clustering.test.configuration; import org.springframework.batch.core.Job; -import org.springframework.batch.core.configuration.annotation.BatchConfigurer; +import org.springframework.batch.core.launch.JobLauncher; +import org.springframework.batch.core.repository.JobRepository; import org.springframework.batch.test.JobLauncherTestUtils; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.context.annotation.Bean; @@ -29,11 +29,6 @@ import uk.ac.ebi.eva.accession.clustering.configuration.batch.recovery.RSAccessionRecoveryServiceConfiguration; import uk.ac.ebi.eva.accession.clustering.configuration.batch.steps.RSAccessionRecoveryStepConfiguration; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.ClusteredVariantAccessioningConfiguration; -import uk.ac.ebi.eva.commons.batch.configuration.SpringBoot1CompatibilityConfiguration; -import uk.ac.ebi.eva.commons.batch.job.JobExecutionApplicationListener; - -import javax.persistence.EntityManagerFactory; -import javax.sql.DataSource; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.RS_ACCESSION_RECOVERY_JOB; @@ -47,35 +42,14 @@ public class RSAccessionRecoveryTestConfiguration { public static final String JOB_LAUNCHER_RS_ACCESSION_RECOVERY = "JOB_LAUNCHER_RS_ACCESSION_RECOVERY"; - @Bean - public BatchConfigurer configurer(DataSource dataSource, EntityManagerFactory entityManagerFactory) - throws Exception { - return SpringBoot1CompatibilityConfiguration.getSpringBoot1CompatibleBatchConfigurer(dataSource, - entityManagerFactory); - } - - @Bean - public JobLauncherTestUtils jobLauncherTestUtils(BatchConfigurer configurer) throws Exception { - JobLauncherTestUtils jobLauncherTestUtils = new JobLauncherTestUtils(); - jobLauncherTestUtils.setJobLauncher(configurer.getJobLauncher()); - jobLauncherTestUtils.setJobRepository(configurer.getJobRepository()); - return jobLauncherTestUtils; - } - - @Bean - public JobExecutionApplicationListener jobExecutionApplicationListener() { - return new JobExecutionApplicationListener(); - } - @Bean(JOB_LAUNCHER_RS_ACCESSION_RECOVERY) - public JobLauncherTestUtils jobLauncherTestUtilsMonotonicAccessionRecoveryAgent() { - - return new JobLauncherTestUtils() { - @Override - @Autowired - public void setJob(@Qualifier(RS_ACCESSION_RECOVERY_JOB) Job job) { - super.setJob(job); - } - }; + public JobLauncherTestUtils jobLauncherTestUtilsMonotonicAccessionRecoveryAgent(JobLauncher jobLauncher, + JobRepository jobRepository, + @Qualifier(RS_ACCESSION_RECOVERY_JOB) Job job) { + JobLauncherTestUtils utils = new JobLauncherTestUtils(); + utils.setJobLauncher(jobLauncher); + utils.setJobRepository(jobRepository); + utils.setJob(job); + return utils; } } \ No newline at end of file diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/test/rule/FixSpringMongoDbRule.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/test/rule/FixSpringMongoDbRule.java deleted file mode 100644 index 001faf1c6..000000000 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/test/rule/FixSpringMongoDbRule.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * - * * Copyright 2020 EMBL - European Bioinformatics Institute - * * - * * Licensed under the Apache License, Version 2.0 (the "License"); - * * you may not use this file except in compliance with the License. - * * You may obtain a copy of the License at - * * - * * http://www.apache.org/licenses/LICENSE-2.0 - * * - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, - * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * See the License for the specific language governing permissions and - * * limitations under the License. - * - */ -package uk.ac.ebi.eva.accession.clustering.test.rule; - -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfiguration; -import com.lordofthejars.nosqlunit.mongodb.SpringMongoDbRule; - -/** - * Temporary fix until nosql unit rc-6 or final is released - */ -public class FixSpringMongoDbRule extends SpringMongoDbRule { - - public FixSpringMongoDbRule(MongoDbConfiguration mongoDbConfiguration) { - super(mongoDbConfiguration); - } - - public FixSpringMongoDbRule(MongoDbConfiguration mongoDbConfiguration, Object object) { - super(mongoDbConfiguration, object); - } - - @Override - public void close() { - // DO NOT CLOSE the connection (Spring will do it when destroying the context) - } -} diff --git a/eva-accession-clustering/src/test/resources/backpropagation-test.properties b/eva-accession-clustering/src/test/resources/backpropagation-test.properties index 3c24d4c6d..336a50626 100644 --- a/eva-accession-clustering/src/test/resources/backpropagation-test.properties +++ b/eva-accession-clustering/src/test/resources/backpropagation-test.properties @@ -23,9 +23,6 @@ accessioning.monotonic.rs.nextBlockInterval=1000000000 spring.data.mongodb.database=test-db -spring.data.mongodb.host=|eva.mongo.host.test| -spring.data.mongodb.password= -spring.data.mongodb.port=27017 mongodb.read-preference=primary # See https://github.com/spring-projects/spring-boot/wiki/Spring-Boot-2.1-Release-Notes#bean-overriding diff --git a/eva-accession-clustering/src/test/resources/clustering-issuance-test.properties b/eva-accession-clustering/src/test/resources/clustering-issuance-test.properties index 5184a2b48..d39fc667d 100644 --- a/eva-accession-clustering/src/test/resources/clustering-issuance-test.properties +++ b/eva-accession-clustering/src/test/resources/clustering-issuance-test.properties @@ -24,9 +24,6 @@ accessioning.monotonic.rs.nextBlockInterval=1000000000 spring.data.mongodb.database=test-db -spring.data.mongodb.host=|eva.mongo.host.test| -spring.data.mongodb.password= -spring.data.mongodb.port=27017 mongodb.read-preference=primary # See https://github.com/spring-projects/spring-boot/wiki/Spring-Boot-2.1-Release-Notes#bean-overriding diff --git a/eva-accession-clustering/src/test/resources/clustering-pipeline-test.properties b/eva-accession-clustering/src/test/resources/clustering-pipeline-test.properties index e11c5f099..cadc78ebb 100644 --- a/eva-accession-clustering/src/test/resources/clustering-pipeline-test.properties +++ b/eva-accession-clustering/src/test/resources/clustering-pipeline-test.properties @@ -6,6 +6,7 @@ parameters.assemblyAccession=GCA_000000001.1 parameters.remappedFrom=ASM1 parameters.rsReportPath=src/test/resources/output-files/rsReport.txt parameters.chunkSize=2 +parameters.rsAccFile=src/test/resources/input-files/rsAcc.txt eva.count-stats.url=http://localhost:8080 eva.count-stats.username=username @@ -23,9 +24,6 @@ accessioning.monotonic.rs.nextBlockInterval=1000000000 spring.data.mongodb.database=test-db -spring.data.mongodb.host=|eva.mongo.host.test| -spring.data.mongodb.password= -spring.data.mongodb.port=27017 mongodb.read-preference=primary # See https://github.com/spring-projects/spring-boot/wiki/Spring-Boot-2.1-Release-Notes#bean-overriding diff --git a/eva-accession-clustering/src/test/resources/clustering-qc-test.properties b/eva-accession-clustering/src/test/resources/clustering-qc-test.properties index 35fad5052..4b654722b 100644 --- a/eva-accession-clustering/src/test/resources/clustering-qc-test.properties +++ b/eva-accession-clustering/src/test/resources/clustering-qc-test.properties @@ -23,9 +23,6 @@ accessioning.monotonic.rs.nextBlockInterval=1000000000 spring.data.mongodb.database=test-db -spring.data.mongodb.host=|eva.mongo.host.test| -spring.data.mongodb.password= -spring.data.mongodb.port=27017 mongodb.read-preference=primary # See https://github.com/spring-projects/spring-boot/wiki/Spring-Boot-2.1-Release-Notes#bean-overriding diff --git a/eva-accession-clustering/src/test/resources/clustering-writer-test.properties b/eva-accession-clustering/src/test/resources/clustering-writer-test.properties index 30cb01327..652d34dd3 100644 --- a/eva-accession-clustering/src/test/resources/clustering-writer-test.properties +++ b/eva-accession-clustering/src/test/resources/clustering-writer-test.properties @@ -24,9 +24,6 @@ accessioning.monotonic.rs.nextBlockInterval=1000000000 spring.data.mongodb.database=test-db -spring.data.mongodb.host=|eva.mongo.host.test| -spring.data.mongodb.password= -spring.data.mongodb.port=27017 mongodb.read-preference=primary # See https://github.com/spring-projects/spring-boot/wiki/Spring-Boot-2.1-Release-Notes#bean-overriding diff --git a/eva-accession-clustering/src/test/resources/duplicate-rs-acc-qc-test.properties b/eva-accession-clustering/src/test/resources/duplicate-rs-acc-qc-test.properties index 7fd3d655b..6c0cf5110 100644 --- a/eva-accession-clustering/src/test/resources/duplicate-rs-acc-qc-test.properties +++ b/eva-accession-clustering/src/test/resources/duplicate-rs-acc-qc-test.properties @@ -25,9 +25,6 @@ accessioning.monotonic.rs.nextBlockInterval=1000000000 spring.data.mongodb.database=test-db -spring.data.mongodb.host=|eva.mongo.host.test| -spring.data.mongodb.password= -spring.data.mongodb.port=27017 mongodb.read-preference=primary # See https://github.com/spring-projects/spring-boot/wiki/Spring-Boot-2.1-Release-Notes#bean-overriding diff --git a/eva-accession-clustering/src/test/resources/merge-split-test.properties b/eva-accession-clustering/src/test/resources/merge-split-test.properties index 8740229b5..3aea699b4 100644 --- a/eva-accession-clustering/src/test/resources/merge-split-test.properties +++ b/eva-accession-clustering/src/test/resources/merge-split-test.properties @@ -22,9 +22,6 @@ accessioning.monotonic.rs.nextBlockInterval=1000000000 spring.data.mongodb.database=test-db -spring.data.mongodb.host=|eva.mongo.host.test| -spring.data.mongodb.password= -spring.data.mongodb.port=27017 mongodb.read-preference=primary # See https://github.com/spring-projects/spring-boot/wiki/Spring-Boot-2.1-Release-Notes#bean-overriding diff --git a/eva-accession-clustering/src/test/resources/rs-accession-recovery.properties b/eva-accession-clustering/src/test/resources/rs-accession-recovery.properties index a66292426..b5418f3f3 100644 --- a/eva-accession-clustering/src/test/resources/rs-accession-recovery.properties +++ b/eva-accession-clustering/src/test/resources/rs-accession-recovery.properties @@ -4,9 +4,10 @@ spring.datasource.driver-class-name=org.hsqldb.jdbcDriver spring.datasource.url=jdbc:hsqldb:mem:db;sql.syntax_pgs=true;DB_CLOSE_DELAY=-1 spring.datasource.username=SA spring.datasource.password= -spring.datasource.schema=test-data/contiguous_id_blocks_schema.sql -spring.datasource.data=test-data/rs_accession_recovery_test_data.sql -spring.jpa.hibernate.ddl-auto=update +spring.sql.init.schema-locations=classpath:test-data/contiguous_id_blocks_schema.sql +spring.sql.init.data-locations=classpath:test-data/rs_accession_recovery_test_data.sql +spring.sql.init.mode=always +spring.jpa.hibernate.ddl-auto=none parameters.vcf=src/test/resources/input-files/vcf/aggregated_accessioned.vcf.gz parameters.projectAccession=projectId_1 @@ -31,9 +32,6 @@ accessioning.monotonic.rs.blockStartValue=3000000000 accessioning.monotonic.rs.nextBlockInterval=1000000000 spring.data.mongodb.database=test-db -spring.data.mongodb.host=|eva.mongo.host.test| -spring.data.mongodb.password= -spring.data.mongodb.port=27017 mongodb.read-preference=primary # See https://github.com/spring-projects/spring-boot/wiki/Spring-Boot-2.1-Release-Notes#bean-overriding diff --git a/eva-accession-core/pom.xml b/eva-accession-core/pom.xml index 3d570aa06..9335ac7ea 100644 --- a/eva-accession-core/pom.xml +++ b/eva-accession-core/pom.xml @@ -5,7 +5,7 @@ uk.ac.ebi.eva eva-accession - 0.6.58-SNAPSHOT + ${revision} eva-accession-core @@ -49,8 +49,18 @@ accession-commons-monotonic-generator-jpa - com.lordofthejars - nosqlunit-mongodb + org.springframework.boot + spring-boot-testcontainers + test + + + org.testcontainers + mongodb + test + + + org.testcontainers + junit-jupiter test @@ -91,8 +101,4 @@ - - localhost - - diff --git a/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/batch/io/ClusteredVariantDeprecationWriter.java b/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/batch/io/ClusteredVariantDeprecationWriter.java index fe0abc075..232e40a37 100644 --- a/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/batch/io/ClusteredVariantDeprecationWriter.java +++ b/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/batch/io/ClusteredVariantDeprecationWriter.java @@ -20,11 +20,12 @@ import org.apache.commons.lang3.tuple.ImmutablePair; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.batch.item.Chunk; import org.springframework.batch.item.ItemWriter; import org.springframework.data.mongodb.core.MongoTemplate; import uk.ac.ebi.ampt2d.commons.accession.core.models.EventType; import uk.ac.ebi.ampt2d.commons.accession.persistence.mongodb.document.EventDocument; - +import uk.ac.ebi.eva.accession.core.EVAObjectModelUtils; import uk.ac.ebi.eva.accession.core.model.IClusteredVariant; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpClusteredVariantEntity; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpClusteredVariantOperationEntity; @@ -33,7 +34,6 @@ import uk.ac.ebi.eva.accession.core.model.eva.ClusteredVariantOperationEntity; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; import uk.ac.ebi.eva.accession.core.service.nonhuman.SubmittedVariantAccessioningService; -import uk.ac.ebi.eva.accession.core.EVAObjectModelUtils; import java.util.Arrays; import java.util.Collections; @@ -67,7 +67,7 @@ public ClusteredVariantDeprecationWriter(String assemblyAccession, MongoTemplate SubmittedVariantAccessioningService submittedVariantAccessioningService, Long accessioningMonotonicInitRs, String deprecationIdSuffix, String deprecationReason) { - ReadPreference readPreference = mongoTemplate.getMongoDbFactory().getDb().getReadPreference(); + ReadPreference readPreference = mongoTemplate.getMongoDatabaseFactory().getMongoDatabase().getReadPreference(); if (!readPreference.equals(ReadPreference.primary())) { throw new IllegalStateException("Read preference setting should be primary to deprecate variants!"); } @@ -84,49 +84,51 @@ public int getNumDeprecatedEntities() { } @Override - public void write(List cvesToDeprecate) { - List cvesToDeprecateInCVE = cvesToDeprecate.stream().filter( + public void write(Chunk cvesToDeprecate) { + List cvesToDeprecateInCVE = cvesToDeprecate.getItems().stream().filter( cve -> (cve.getAccession() >= accessioningMonotonicInitRs)).collect(Collectors.toList()); - List cvesToDeprecateInDbsnpCVE = cvesToDeprecate.stream().filter( + List cvesToDeprecateInDbsnpCVE = cvesToDeprecate.getItems().stream().filter( cve -> (cve.getAccession() < accessioningMonotonicInitRs)).collect(Collectors.toList()); deprecateVariants(cvesToDeprecateInCVE, ClusteredVariantEntity.class); deprecateVariants(cvesToDeprecateInDbsnpCVE, DbsnpClusteredVariantEntity.class); } private void deprecateVariants(List cvesToDeprecate, - Class cveCollectionToUse) { + Class cveCollectionToUse) { if (cvesToDeprecate.size() > 0) { Set> rsHashesAndIDsToRemove = cvesToDeprecate.stream().map( - cve -> new ImmutablePair<>(cve.getHashedMessage(), cve.getAccession())).collect(Collectors.toSet()); + cve -> new ImmutablePair<>(cve.getHashedMessage(), cve.getAccession())) + .collect(Collectors.toSet()); Set> rsHashesAndIDsAssociatedWithExistingSS = this.submittedVariantAccessioningService.getByClusteredVariantAccessionIn( - rsHashesAndIDsToRemove.stream().map(c -> c.right).collect(Collectors.toList())).stream() + rsHashesAndIDsToRemove.stream().map(c -> c.right) + .collect(Collectors.toList())).stream() .map(result -> new SubmittedVariantEntity(result.getAccession(), result.getHash(), result.getData(), result.getVersion())) .filter(result -> result.getReferenceSequenceAccession().equals(this.assemblyAccession) && rsHashesAndIDsToRemove.contains( - new ImmutablePair<>(EVAObjectModelUtils.getClusteredVariantHash(result), - result.getClusteredVariantAccession()))) + new ImmutablePair<>(EVAObjectModelUtils.getClusteredVariantHash(result), + result.getClusteredVariantAccession()))) .map(result -> new ImmutablePair<>(EVAObjectModelUtils.getClusteredVariantHash(result), result.getClusteredVariantAccession())) .collect(Collectors.toSet()); if (rsHashesAndIDsAssociatedWithExistingSS.size() > 0) { logger.warn("The following RS ID/hash combinations are still associated with existing submitted variants. " + - "Hence they will not be deprecated. The combinations are: " + + "Hence they will not be deprecated. The combinations are: " + rsHashesAndIDsAssociatedWithExistingSS.stream().map(Object::toString).collect( Collectors.joining(","))); } rsHashesAndIDsToRemove.removeAll(rsHashesAndIDsAssociatedWithExistingSS); cvesToDeprecate = cvesToDeprecate.stream() - .filter(cve -> - rsHashesAndIDsToRemove.contains(new ImmutablePair<>( - cve.getHashedMessage(), cve.getAccession()))) - .collect(Collectors.toList()); + .filter(cve -> + rsHashesAndIDsToRemove.contains(new ImmutablePair<>( + cve.getHashedMessage(), cve.getAccession()))) + .collect(Collectors.toList()); Set rsHashesToRemove = cvesToDeprecate.stream() - .map(ClusteredVariantEntity::getId) - .collect(Collectors.toSet()); + .map(ClusteredVariantEntity::getId) + .collect(Collectors.toSet()); Class> cvoeCollectionToUse = cveCollectionToUse.equals(ClusteredVariantEntity.class) ? ClusteredVariantOperationEntity.class : DbsnpClusteredVariantOperationEntity.class; @@ -138,20 +140,20 @@ private void deprecateVariants(List cvesToDepr } private void writeDeprecationOperation(List cvesToDeprecate, - Class> cvoeCollectionToUse) { + Class> cvoeCollectionToUse) { List cvoesToWrite = cvesToDeprecate.stream().map(cve -> { ClusteredVariantOperationEntity cvoe = new ClusteredVariantOperationEntity(); cvoe.fill(EventType.DEPRECATED, cve.getAccession(), null, - this.deprecationReason, - Collections.singletonList(new ClusteredVariantInactiveEntity(cve))); + this.deprecationReason, + Collections.singletonList(new ClusteredVariantInactiveEntity(cve))); cvoe.setId(String.join("_", Arrays.asList("RS_DEPRECATED", this.deprecationIdSuffix, cve.getId()))); return cvoe; }).collect(Collectors.toList()); Set operationIdsToWrite = cvoesToWrite.stream().map(EventDocument::getId).collect(Collectors.toSet()); Set alreadyExistingIds = this.mongoTemplate.find(query(where("_id").in(operationIdsToWrite)), - cvoeCollectionToUse).stream().map(EventDocument::getId) - .collect(Collectors.toSet()); + cvoeCollectionToUse).stream().map(EventDocument::getId) + .collect(Collectors.toSet()); operationIdsToWrite.removeAll(alreadyExistingIds); cvoesToWrite = cvoesToWrite.stream().filter(svoe -> operationIdsToWrite.contains(svoe.getId())).collect( Collectors.toList()); diff --git a/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/batch/io/DbsnpClusteredVariantOperationWriter.java b/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/batch/io/DbsnpClusteredVariantOperationWriter.java index 992c3f4c3..6c5097725 100644 --- a/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/batch/io/DbsnpClusteredVariantOperationWriter.java +++ b/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/batch/io/DbsnpClusteredVariantOperationWriter.java @@ -20,15 +20,13 @@ import com.mongodb.MongoBulkWriteException; import com.mongodb.bulk.BulkWriteResult; +import org.springframework.batch.item.Chunk; import org.springframework.batch.item.ItemWriter; import org.springframework.dao.DuplicateKeyException; import org.springframework.data.mongodb.core.BulkOperations; import org.springframework.data.mongodb.core.MongoTemplate; - -import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpClusteredVariantOperationEntity; import uk.ac.ebi.eva.accession.core.batch.listeners.ImportCounts; - -import java.util.List; +import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpClusteredVariantOperationEntity; public class DbsnpClusteredVariantOperationWriter implements ItemWriter { @@ -42,12 +40,12 @@ public DbsnpClusteredVariantOperationWriter(MongoTemplate mongoTemplate, ImportC } @Override - public void write(List importedClusteredVariantsOperations) + public void write(Chunk importedClusteredVariantsOperations) throws Exception { try { BulkOperations bulkOperations = mongoTemplate.bulkOps(BulkOperations.BulkMode.UNORDERED, - DbsnpClusteredVariantOperationEntity.class); - bulkOperations.insert(importedClusteredVariantsOperations); + DbsnpClusteredVariantOperationEntity.class); + bulkOperations.insert(importedClusteredVariantsOperations.getItems()); bulkOperations.execute(); importCounts.addOperationsWritten(importedClusteredVariantsOperations.size()); } catch (DuplicateKeyException exception) { diff --git a/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/batch/io/DbsnpClusteredVariantWriter.java b/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/batch/io/DbsnpClusteredVariantWriter.java index 65f94de05..0231a7b29 100644 --- a/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/batch/io/DbsnpClusteredVariantWriter.java +++ b/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/batch/io/DbsnpClusteredVariantWriter.java @@ -17,14 +17,13 @@ import com.mongodb.MongoBulkWriteException; import com.mongodb.bulk.BulkWriteResult; +import org.springframework.batch.item.Chunk; import org.springframework.batch.item.ItemWriter; -import org.springframework.dao.DuplicateKeyException; +import org.springframework.data.mongodb.BulkOperationException; import org.springframework.data.mongodb.core.BulkOperations; import org.springframework.data.mongodb.core.MongoTemplate; -import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpClusteredVariantEntity; import uk.ac.ebi.eva.accession.core.batch.listeners.ImportCounts; - -import java.util.List; +import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpClusteredVariantEntity; public class DbsnpClusteredVariantWriter implements ItemWriter { @@ -38,14 +37,14 @@ public DbsnpClusteredVariantWriter(MongoTemplate mongoTemplate, ImportCounts imp } @Override - public void write(List importedClusteredVariants) { + public void write(Chunk importedClusteredVariants) { try { BulkOperations bulkOperations = mongoTemplate.bulkOps(BulkOperations.BulkMode.UNORDERED, - DbsnpClusteredVariantEntity.class); - bulkOperations.insert(importedClusteredVariants); + DbsnpClusteredVariantEntity.class); + bulkOperations.insert(importedClusteredVariants.getItems()); bulkOperations.execute(); importCounts.addClusteredVariantsWritten(importedClusteredVariants.size()); - } catch (DuplicateKeyException exception) { + } catch (BulkOperationException exception) { BulkWriteResult bulkWriteResult = ((MongoBulkWriteException) exception.getCause()).getWriteResult(); importCounts.addClusteredVariantsWritten(bulkWriteResult.getInsertedCount()); throw exception; diff --git a/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/batch/io/SubmittedVariantDeprecationWriter.java b/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/batch/io/SubmittedVariantDeprecationWriter.java index d7ef84dd2..10e22bf82 100644 --- a/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/batch/io/SubmittedVariantDeprecationWriter.java +++ b/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/batch/io/SubmittedVariantDeprecationWriter.java @@ -18,12 +18,13 @@ import com.mongodb.ReadPreference; import org.apache.commons.lang3.tuple.ImmutablePair; +import org.springframework.batch.item.Chunk; import org.springframework.batch.item.ItemWriter; import org.springframework.data.mongodb.core.MongoTemplate; import uk.ac.ebi.ampt2d.commons.accession.core.models.EventType; import uk.ac.ebi.ampt2d.commons.accession.persistence.mongodb.document.AccessionedDocument; import uk.ac.ebi.ampt2d.commons.accession.persistence.mongodb.document.EventDocument; - +import uk.ac.ebi.eva.accession.core.EVAObjectModelUtils; import uk.ac.ebi.eva.accession.core.model.ISubmittedVariant; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpSubmittedVariantEntity; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpSubmittedVariantOperationEntity; @@ -33,9 +34,12 @@ import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantOperationEntity; import uk.ac.ebi.eva.accession.core.service.nonhuman.ClusteredVariantAccessioningService; import uk.ac.ebi.eva.accession.core.service.nonhuman.SubmittedVariantAccessioningService; -import uk.ac.ebi.eva.accession.core.EVAObjectModelUtils; -import java.util.*; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.Set; import java.util.stream.Collectors; import static org.springframework.data.mongodb.core.query.Criteria.where; @@ -64,7 +68,7 @@ public SubmittedVariantDeprecationWriter(String assemblyAccession, MongoTemplate ClusteredVariantAccessioningService clusteredVariantAccessioningService, Long accessioningMonotonicInitSs, Long accessioningMonotonicInitRs, String deprecationIdSuffix, String deprecationReason) { - ReadPreference readPreference = mongoTemplate.getMongoDbFactory().getDb().getReadPreference(); + ReadPreference readPreference = mongoTemplate.getMongoDatabaseFactory().getMongoDatabase().getReadPreference(); if (!readPreference.equals(ReadPreference.primary())) { throw new IllegalStateException("Read preference setting should be primary to deprecate variants!"); } @@ -76,25 +80,27 @@ public SubmittedVariantDeprecationWriter(String assemblyAccession, MongoTemplate this.deprecationReason = deprecationReason; this.clusteredVariantDeprecationWriter = new ClusteredVariantDeprecationWriter(this.assemblyAccession, - this.mongoTemplate, - submittedVariantAccessioningService, - accessioningMonotonicInitRs, - this.deprecationIdSuffix, - this.deprecationReason); + this.mongoTemplate, + submittedVariantAccessioningService, + accessioningMonotonicInitRs, + this.deprecationIdSuffix, + this.deprecationReason); } @Override - public void write(List svesToDeprecate) { - List svesToDeprecateInSVE = svesToDeprecate.stream().filter( - sve -> (sve.getAccession() >= accessioningMonotonicInitSs)).collect(Collectors.toList()); - List svesToDeprecateInDbsnpSVE = svesToDeprecate.stream().filter( - sve -> (sve.getAccession() < accessioningMonotonicInitSs)).collect(Collectors.toList()); + public void write(Chunk svesToDeprecate) { + List svesToDeprecateInSVE = svesToDeprecate.getItems().stream().filter( + sve -> (sve.getAccession() >= accessioningMonotonicInitSs)) + .collect(Collectors.toList()); + List svesToDeprecateInDbsnpSVE = svesToDeprecate.getItems().stream().filter( + sve -> (sve.getAccession() < accessioningMonotonicInitSs)) + .collect(Collectors.toList()); deprecateVariants(svesToDeprecateInSVE, SubmittedVariantEntity.class); deprecateVariants(svesToDeprecateInDbsnpSVE, DbsnpSubmittedVariantEntity.class); } private void deprecateVariants(List svesToDeprecate, - Class sveCollectionToUse) { + Class sveCollectionToUse) { if (svesToDeprecate.size() > 0) { List ssHashesToRemove = svesToDeprecate.stream().map(AccessionedDocument::getId).collect( Collectors.toList()); @@ -111,7 +117,7 @@ private void deprecateVariants(List svesToDepr associatedRSHashesAndIDs.contains( new ImmutablePair<>(result.getHash(), result.getAccession()))) .map(result -> new ClusteredVariantEntity(result.getAccession(), result.getHash(), - result.getData())) + result.getData())) .collect(Collectors.toList()); Class> svoeCollectionToUse = sveCollectionToUse.equals(SubmittedVariantEntity.class) ? @@ -120,25 +126,25 @@ private void deprecateVariants(List svesToDepr List removedEntities = this.mongoTemplate.findAllAndRemove( query(where("_id").in(ssHashesToRemove)), sveCollectionToUse); this.numDeprecatedSubmittedEntities += removedEntities.size(); - this.clusteredVariantDeprecationWriter.write(cvesToDeprecate); + this.clusteredVariantDeprecationWriter.write(new Chunk<>(cvesToDeprecate)); } } private void writeDeprecationOperation(List svesToDeprecate, - Class> svoeCollectionToUse) { + Class> svoeCollectionToUse) { List svoesToWrite = svesToDeprecate.stream().map(sve -> { SubmittedVariantOperationEntity svoe = new SubmittedVariantOperationEntity(); svoe.fill(EventType.DEPRECATED, sve.getAccession(), null, - this.deprecationReason, - Collections.singletonList(new SubmittedVariantInactiveEntity(sve))); + this.deprecationReason, + Collections.singletonList(new SubmittedVariantInactiveEntity(sve))); svoe.setId(String.join("_", Arrays.asList("SS_DEPRECATED", this.deprecationIdSuffix, sve.getId()))); return svoe; }).collect(Collectors.toList()); Set operationIds = svoesToWrite.stream().map(EventDocument::getId).collect(Collectors.toSet()); Set alreadyExistingIds = this.mongoTemplate.find(query(where("_id").in(operationIds)), - svoeCollectionToUse).stream().map(EventDocument::getId) - .collect(Collectors.toSet()); + svoeCollectionToUse).stream().map(EventDocument::getId) + .collect(Collectors.toSet()); operationIds.removeAll(alreadyExistingIds); svoesToWrite = svoesToWrite.stream().filter(svoe -> operationIds.contains(svoe.getId())).collect( Collectors.toList()); diff --git a/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/batch/listeners/GenericProgressListener.java b/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/batch/listeners/GenericProgressListener.java index e14abd58a..5d6c1fed2 100644 --- a/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/batch/listeners/GenericProgressListener.java +++ b/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/batch/listeners/GenericProgressListener.java @@ -22,8 +22,7 @@ import org.springframework.batch.core.StepExecution; import org.springframework.batch.core.listener.StepListenerSupport; import org.springframework.batch.core.scope.context.ChunkContext; - -import java.util.List; +import org.springframework.batch.item.Chunk; public class GenericProgressListener extends StepListenerSupport { @@ -64,14 +63,14 @@ public void afterRead(I itemRead) { } @Override - public void beforeWrite(List items) { + public void beforeWrite(Chunk items) { logger.debug("About to write chunk"); } @Override - public void afterWrite(List items) { + public void afterWrite(Chunk items) { if (items.size() > 0) { - O lastItem = items.get(items.size() - 1); + O lastItem = items.getItems().get(items.size() - 1); logger.debug("Written chunk of {} items. Last item was {}: {}", items.size(), lastItem.toString()); } else { logger.debug("Written chunk of 0 items."); diff --git a/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/batch/policies/IllegalStartSkipPolicy.java b/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/batch/policies/IllegalStartSkipPolicy.java index a4f8c5a92..e40490c15 100644 --- a/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/batch/policies/IllegalStartSkipPolicy.java +++ b/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/batch/policies/IllegalStartSkipPolicy.java @@ -23,7 +23,7 @@ public class IllegalStartSkipPolicy implements SkipPolicy { @Override - public boolean shouldSkip(Throwable exception, int skipCount) throws SkipLimitExceededException { + public boolean shouldSkip(Throwable exception, long skipCount) throws SkipLimitExceededException { return exception instanceof PositionOutsideOfContigException; } } diff --git a/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/configuration/MongoClientCreator.java b/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/configuration/MongoClientCreator.java index 221e1148c..48fc310ed 100644 --- a/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/configuration/MongoClientCreator.java +++ b/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/configuration/MongoClientCreator.java @@ -1,41 +1,44 @@ package uk.ac.ebi.eva.accession.core.configuration; -import com.mongodb.*; +import com.mongodb.ConnectionString; +import com.mongodb.MongoClientSettings; +import com.mongodb.ReadConcern; +import com.mongodb.ReadPreference; +import com.mongodb.WriteConcern; +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoClients; import org.springframework.beans.factory.ObjectProvider; -import org.springframework.boot.autoconfigure.mongo.MongoClientFactory; import org.springframework.boot.autoconfigure.mongo.MongoProperties; -import org.springframework.core.env.Environment; import uk.ac.ebi.eva.commons.mongodb.utils.MongoUtils; -import java.io.UnsupportedEncodingException; import java.util.Objects; public class MongoClientCreator { - public static MongoClient getMongoClient(MongoProperties properties, ObjectProvider options, - Environment environment, String readPreference) - throws UnsupportedEncodingException { - MongoClientOptions mongoClientOptions = options.getIfAvailable(); + public static MongoClient getMongoClient(MongoProperties properties, ObjectProvider settings, + String readPreference) { + MongoClientSettings mongoClientSettings = settings.getIfAvailable(); // Only set the URI if it isn't already set if (Objects.isNull(properties.getUri())) { // Weirdly, MongoClient instantiation works without authentication mechanism // in the eva-accession project but does not work in the eva-pipeline project // So we explicitly pass it as null here - properties.setUri(MongoUtils.constructMongoClientURI(properties.getHost(), properties.getPort(), + properties.setUri(MongoUtils.constructMongoConnectionString(properties.getHost(), properties.getPort(), properties.getDatabase(), properties.getUsername(), (Objects.nonNull(properties.getPassword()) ? String.valueOf(properties.getPassword()) : ""), - properties.getAuthenticationDatabase(), null, readPreference).getURI()); + properties.getAuthenticationDatabase(), null, readPreference).getConnectionString()); } // If we don't do this Spring gets confused since both the URI (which already has the host) // and the host parameters are present properties.setHost(null); - MongoClientOptions.Builder mongoClientOptionsBuilder; - if (mongoClientOptions != null) { - mongoClientOptionsBuilder = new MongoClientOptions.Builder(mongoClientOptions); + MongoClientSettings.Builder mongoClientSettingsBuilder; + if (mongoClientSettings != null) { + mongoClientSettingsBuilder = MongoClientSettings.builder(mongoClientSettings); } else { - mongoClientOptionsBuilder = new MongoClientOptions.Builder(); + mongoClientSettingsBuilder = MongoClientSettings.builder(); } - mongoClientOptions = mongoClientOptionsBuilder.readPreference(ReadPreference.valueOf(readPreference)) - .writeConcern(WriteConcern.MAJORITY).readConcern(ReadConcern.MAJORITY).build(); - return new MongoClientFactory(properties, environment).createMongoClient(mongoClientOptions); + mongoClientSettings = mongoClientSettingsBuilder.readPreference(ReadPreference.valueOf(readPreference)) + .writeConcern(WriteConcern.MAJORITY).readConcern(ReadConcern.MAJORITY) + .applyConnectionString(new ConnectionString(properties.getUri())).build(); + return MongoClients.create(mongoClientSettings); } } diff --git a/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/configuration/human/HumanMongoConfiguration.java b/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/configuration/human/HumanMongoConfiguration.java index 847d0ad24..5876c486c 100644 --- a/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/configuration/human/HumanMongoConfiguration.java +++ b/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/configuration/human/HumanMongoConfiguration.java @@ -15,8 +15,8 @@ */ package uk.ac.ebi.eva.accession.core.configuration.human; -import com.mongodb.MongoClient; -import com.mongodb.MongoClientOptions; +import com.mongodb.MongoClientSettings; +import com.mongodb.client.MongoClient; import org.springframework.beans.factory.ObjectProvider; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Value; @@ -25,12 +25,11 @@ import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import org.springframework.core.env.Environment; import org.springframework.data.domain.Sort; -import org.springframework.data.mongodb.MongoDbFactory; +import org.springframework.data.mongodb.MongoDatabaseFactory; import org.springframework.data.mongodb.config.EnableMongoAuditing; import org.springframework.data.mongodb.core.MongoTemplate; -import org.springframework.data.mongodb.core.SimpleMongoDbFactory; +import org.springframework.data.mongodb.core.SimpleMongoClientDatabaseFactory; import org.springframework.data.mongodb.core.WriteResultChecking; import org.springframework.data.mongodb.core.convert.DefaultDbRefResolver; import org.springframework.data.mongodb.core.convert.DefaultMongoTypeMapper; @@ -63,30 +62,28 @@ public MongoProperties mongoProperties() { @Bean("humanMongoClient") public MongoClient mongoClient(@Qualifier("humanMongoProperties") MongoProperties properties, - ObjectProvider options, Environment environment) + ObjectProvider settings) throws UnknownHostException, UnsupportedEncodingException { - return MongoClientCreator.getMongoClient(properties, options, environment, readPreference); + return MongoClientCreator.getMongoClient(properties, settings, readPreference); } @Bean("humanFactory") - public MongoDbFactory mongoDbFactory(@Qualifier("humanMongoProperties") MongoProperties properties, - ObjectProvider options, - Environment environment) + public MongoDatabaseFactory mongoDbFactory(@Qualifier("humanMongoProperties") MongoProperties properties, + ObjectProvider settings) throws UnknownHostException, UnsupportedEncodingException { - return new SimpleMongoDbFactory(mongoClient(properties, options, environment), properties.getDatabase()); + return new SimpleMongoClientDatabaseFactory(mongoClient(properties, settings), properties.getDatabase()); } @Bean("humanMappingConverter") public MappingMongoConverter mappingMongoConverter(MongoProperties properties, - ObjectProvider options, - Environment environment) + ObjectProvider settings) throws UnknownHostException, UnsupportedEncodingException { - return new MappingMongoConverter(new DefaultDbRefResolver(mongoDbFactory(properties, options, environment)), - new MongoMappingContext()); + return new MappingMongoConverter(new DefaultDbRefResolver(mongoDbFactory(properties, settings)), + new MongoMappingContext()); } @Bean(name = "humanMongoTemplate") - public MongoTemplate humanMongoTemplate(@Qualifier("humanFactory") MongoDbFactory mongoDbFactory, + public MongoTemplate humanMongoTemplate(@Qualifier("humanFactory") MongoDatabaseFactory mongoDbFactory, @Qualifier("humanMappingConverter") MappingMongoConverter converter) { converter.setTypeMapper(new DefaultMongoTypeMapper(null)); MongoTemplate mongoTemplate = new MongoTemplate(mongoDbFactory, converter); diff --git a/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/configuration/nonhuman/MongoConfiguration.java b/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/configuration/nonhuman/MongoConfiguration.java index a4c6864f7..4c44b31f4 100644 --- a/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/configuration/nonhuman/MongoConfiguration.java +++ b/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/configuration/nonhuman/MongoConfiguration.java @@ -15,11 +15,10 @@ */ package uk.ac.ebi.eva.accession.core.configuration.nonhuman; -import com.mongodb.MongoClient; -import com.mongodb.MongoClientOptions; - +import com.mongodb.MongoClientSettings; import com.mongodb.ReadPreference; import com.mongodb.WriteConcern; +import com.mongodb.client.MongoClient; import org.springframework.beans.factory.ObjectProvider; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Value; @@ -29,21 +28,22 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Primary; -import org.springframework.core.env.Environment; -import org.springframework.data.mongodb.MongoDbFactory; +import org.springframework.data.mongodb.MongoDatabaseFactory; import org.springframework.data.mongodb.config.EnableMongoAuditing; import org.springframework.data.mongodb.core.MongoTemplate; -import org.springframework.data.mongodb.core.SimpleMongoDbFactory; +import org.springframework.data.mongodb.core.SimpleMongoClientDatabaseFactory; import org.springframework.data.mongodb.core.WriteResultChecking; import org.springframework.data.mongodb.core.convert.DefaultDbRefResolver; import org.springframework.data.mongodb.core.convert.DefaultMongoTypeMapper; import org.springframework.data.mongodb.core.convert.MappingMongoConverter; +import org.springframework.data.mongodb.core.convert.MongoCustomConversions; import org.springframework.data.mongodb.core.mapping.MongoMappingContext; import org.springframework.data.mongodb.repository.config.EnableMongoRepositories; import uk.ac.ebi.eva.accession.core.configuration.MongoClientCreator; import java.io.UnsupportedEncodingException; import java.net.UnknownHostException; +import java.util.Collections; @Configuration @EnableMongoRepositories(basePackages = {"uk.ac.ebi.eva.accession.core.repository"}) @@ -66,33 +66,50 @@ public MongoProperties mongoProperties() { @Bean @Primary - public MongoClient mongoClient(MongoProperties properties, ObjectProvider options, - Environment environment) throws UnknownHostException, UnsupportedEncodingException { - return MongoClientCreator.getMongoClient(properties, options, environment, readPreference); + public MongoClient mongoClient(MongoProperties properties, ObjectProvider settings) + throws UnknownHostException, UnsupportedEncodingException { + return MongoClientCreator.getMongoClient(properties, settings, readPreference); } @Bean("primaryFactory") @Primary - public MongoDbFactory mongoDbFactory(MongoProperties properties, - ObjectProvider options, - Environment environment) + public MongoDatabaseFactory mongoDbFactory(MongoProperties properties, ObjectProvider settings) throws UnknownHostException, UnsupportedEncodingException { - return new SimpleMongoDbFactory(mongoClient(properties, options, environment), properties.getDatabase()); + return new SimpleMongoClientDatabaseFactory(mongoClient(properties, settings), properties.getDatabase()); + } + + @Bean + @Primary + public MongoCustomConversions mongoCustomConversions() { + return new MongoCustomConversions(Collections.emptyList()); + } + + @Bean + @Primary + public MongoMappingContext mongoMappingContext(MongoCustomConversions conversions) { + MongoMappingContext context = new MongoMappingContext(); + context.setSimpleTypeHolder(conversions.getSimpleTypeHolder()); + context.setAutoIndexCreation(false); + return context; } @Bean @Primary public MappingMongoConverter mappingMongoConverter(MongoProperties properties, - ObjectProvider options, - Environment environment) + ObjectProvider settings, + MongoMappingContext mongoMappingContext, + MongoCustomConversions conversions) throws UnknownHostException, UnsupportedEncodingException { - return new MappingMongoConverter(new DefaultDbRefResolver(mongoDbFactory(properties, options, environment)), - new MongoMappingContext()); + MappingMongoConverter converter = new MappingMongoConverter(new DefaultDbRefResolver(mongoDbFactory(properties, settings)), + mongoMappingContext); + converter.setCustomConversions(conversions); + converter.afterPropertiesSet(); + return converter; } @Primary @Bean - public MongoTemplate mongoTemplate(@Qualifier("primaryFactory") MongoDbFactory mongoDbFactory, + public MongoTemplate mongoTemplate(@Qualifier("primaryFactory") MongoDatabaseFactory mongoDbFactory, MappingMongoConverter converter) { converter.setTypeMapper(new DefaultMongoTypeMapper(null)); MongoTemplate mongoTemplate = new MongoTemplate(mongoDbFactory, converter); diff --git a/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/configuration/nonhuman/SubmittedVariantAccessioningConfiguration.java b/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/configuration/nonhuman/SubmittedVariantAccessioningConfiguration.java index e0e1c7840..5e9e17e98 100644 --- a/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/configuration/nonhuman/SubmittedVariantAccessioningConfiguration.java +++ b/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/configuration/nonhuman/SubmittedVariantAccessioningConfiguration.java @@ -24,31 +24,31 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Import; +import org.springframework.context.annotation.Lazy; import uk.ac.ebi.ampt2d.commons.accession.autoconfigure.EnableSpringDataContiguousIdService; import uk.ac.ebi.ampt2d.commons.accession.core.AccessionSaveMode; import uk.ac.ebi.ampt2d.commons.accession.generators.monotonic.MonotonicAccessionGenerator; import uk.ac.ebi.ampt2d.commons.accession.hashing.SHA1HashingFunction; import uk.ac.ebi.ampt2d.commons.accession.persistence.jpa.monotonic.service.ContiguousIdBlockService; - import uk.ac.ebi.eva.accession.core.configuration.ApplicationProperties; import uk.ac.ebi.eva.accession.core.configuration.ApplicationPropertiesConfiguration; import uk.ac.ebi.eva.accession.core.configuration.ContigAliasConfiguration; import uk.ac.ebi.eva.accession.core.contigalias.ContigAliasService; -import uk.ac.ebi.eva.accession.core.model.ISubmittedVariant; -import uk.ac.ebi.eva.accession.core.service.nonhuman.SubmittedVariantAccessioningService; import uk.ac.ebi.eva.accession.core.generators.DbsnpMonotonicAccessionGenerator; -import uk.ac.ebi.eva.accession.core.service.nonhuman.dbsnp.DbsnpSubmittedVariantAccessioningDatabaseService; -import uk.ac.ebi.eva.accession.core.repository.nonhuman.dbsnp.DbsnpSubmittedVariantAccessioningRepository; +import uk.ac.ebi.eva.accession.core.model.ISubmittedVariant; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpSubmittedVariantInactiveEntity; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpSubmittedVariantOperationEntity; -import uk.ac.ebi.eva.accession.core.repository.nonhuman.dbsnp.DbsnpSubmittedVariantOperationRepository; -import uk.ac.ebi.eva.accession.core.service.nonhuman.eva.SubmittedVariantAccessioningDatabaseService; -import uk.ac.ebi.eva.accession.core.repository.nonhuman.eva.SubmittedVariantAccessioningRepository; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantInactiveEntity; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantOperationEntity; +import uk.ac.ebi.eva.accession.core.repository.nonhuman.dbsnp.DbsnpSubmittedVariantAccessioningRepository; +import uk.ac.ebi.eva.accession.core.repository.nonhuman.dbsnp.DbsnpSubmittedVariantOperationRepository; +import uk.ac.ebi.eva.accession.core.repository.nonhuman.eva.SubmittedVariantAccessioningRepository; import uk.ac.ebi.eva.accession.core.repository.nonhuman.eva.SubmittedVariantOperationRepository; +import uk.ac.ebi.eva.accession.core.service.nonhuman.SubmittedVariantAccessioningService; +import uk.ac.ebi.eva.accession.core.service.nonhuman.dbsnp.DbsnpSubmittedVariantAccessioningDatabaseService; import uk.ac.ebi.eva.accession.core.service.nonhuman.dbsnp.DbsnpSubmittedVariantInactiveService; import uk.ac.ebi.eva.accession.core.service.nonhuman.dbsnp.DbsnpSubmittedVariantMonotonicAccessioningService; +import uk.ac.ebi.eva.accession.core.service.nonhuman.eva.SubmittedVariantAccessioningDatabaseService; import uk.ac.ebi.eva.accession.core.service.nonhuman.eva.SubmittedVariantInactiveService; import uk.ac.ebi.eva.accession.core.service.nonhuman.eva.SubmittedVariantMonotonicAccessioningService; import uk.ac.ebi.eva.accession.core.summary.SubmittedVariantSummaryFunction; @@ -72,9 +72,11 @@ public class SubmittedVariantAccessioningConfiguration { @Autowired private DbsnpSubmittedVariantOperationRepository dbsnpOperationRepository; + @Lazy @Autowired private SubmittedVariantInactiveService inactiveService; + @Lazy @Autowired private DbsnpSubmittedVariantInactiveService dbsnpInactiveService; diff --git a/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/model/ClusteredVariant.java b/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/model/ClusteredVariant.java index 533baf811..30e7c8091 100644 --- a/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/model/ClusteredVariant.java +++ b/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/model/ClusteredVariant.java @@ -17,11 +17,11 @@ */ package uk.ac.ebi.eva.accession.core.model; +import uk.ac.ebi.eva.commons.core.models.VariantType; + import java.time.LocalDateTime; import java.util.Objects; -import uk.ac.ebi.eva.commons.core.models.VariantType; - public class ClusteredVariant implements IClusteredVariant { private String assemblyAccession; diff --git a/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/model/dbsnp/DbsnpSubmittedVariantInactiveEntity.java b/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/model/dbsnp/DbsnpSubmittedVariantInactiveEntity.java index 83744bd2a..0c45081ea 100644 --- a/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/model/dbsnp/DbsnpSubmittedVariantInactiveEntity.java +++ b/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/model/dbsnp/DbsnpSubmittedVariantInactiveEntity.java @@ -17,7 +17,6 @@ */ package uk.ac.ebi.eva.accession.core.model.dbsnp; -import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpSubmittedVariantEntity; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantInactiveEntity; public class DbsnpSubmittedVariantInactiveEntity extends SubmittedVariantInactiveEntity { diff --git a/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/runner/CommandLineRunnerUtils.java b/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/runner/CommandLineRunnerUtils.java index fc6226c7b..589886bf4 100644 --- a/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/runner/CommandLineRunnerUtils.java +++ b/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/runner/CommandLineRunnerUtils.java @@ -16,6 +16,7 @@ import uk.ac.ebi.eva.commons.batch.job.JobStatusManager; import java.util.Comparator; +import java.util.HashMap; import java.util.List; import java.util.Map; @@ -26,11 +27,11 @@ public class CommandLineRunnerUtils { private static final String RUN_ID_PARAMETER_NAME = "run.id"; public static JobParameters addRunIDToJobParameters(String jobName, JobExplorer jobExplorer, - JobParameters jobParameters) { + JobParameters jobParameters) { JobExecution lastJobExecution = getLastJobExecution(jobName, jobExplorer, jobParameters); if (lastJobExecution != null) { Long runIdParameterFromLastExecution = lastJobExecution.getJobParameters() - .getLong(RUN_ID_PARAMETER_NAME); + .getLong(RUN_ID_PARAMETER_NAME); if (runIdParameterFromLastExecution != 0 && lastJobExecution.getStatus() == BatchStatus.FAILED) { // Spring Batch 4 uses all job parameters (including run.id) to detect previous instances of a job - see https://github.com/spring-projects/spring-boot/blob/86fb39d5c5f474fe3544159270d4c4e2d01d43ef/spring-boot-project/spring-boot-autoconfigure/src/main/java/org/springframework/boot/autoconfigure/batch/JobLauncherCommandLineRunner.java#L222 // as opposed to Spring 3 which uses only job name - see https://github.com/spring-projects/spring-boot/blob/541890f0e003a6e346f2234102c97105ab1292ee/spring-boot-autoconfigure/src/main/java/org/springframework/boot/autoconfigure/batch/JobLauncherCommandLineRunner.java#L131 @@ -43,16 +44,16 @@ public static JobParameters addRunIDToJobParameters(String jobName, JobExplorer return jobParameters; } - public static JobExecution getLastJobExecution (String jobName, JobExplorer jobExplorer, - JobParameters previousJobParameters) { - int previousJobInstanceCount = getPreviousJobInstanceCount(jobName, jobExplorer); + public static JobExecution getLastJobExecution(String jobName, JobExplorer jobExplorer, + JobParameters previousJobParameters) { + long previousJobInstanceCount = getPreviousJobInstanceCount(jobName, jobExplorer); - List jobInstanceList = jobExplorer.getJobInstances(jobName, 0, previousJobInstanceCount); + List jobInstanceList = jobExplorer.getJobInstances(jobName, 0, (int) previousJobInstanceCount); List matchingJobExecutions; - for (JobInstance jobInstance: jobInstanceList) { + for (JobInstance jobInstance : jobInstanceList) { matchingJobExecutions = jobExplorer.getJobExecutions(jobInstance); - for (JobExecution jobExecution: matchingJobExecutions) { + for (JobExecution jobExecution : matchingJobExecutions) { if (areParametersEquivalentExceptRunId(jobExecution.getJobParameters(), previousJobParameters)) { return matchingJobExecutions.stream().max(Comparator.comparingLong(Entity::getId)).get(); } @@ -61,18 +62,17 @@ public static JobExecution getLastJobExecution (String jobName, JobExplorer jobE return null; } - private static int getPreviousJobInstanceCount(String jobName, JobExplorer jobExplorer) { + private static long getPreviousJobInstanceCount(String jobName, JobExplorer jobExplorer) { try { return jobExplorer.getJobInstanceCount(jobName); - } - catch (NoSuchJobException ex) { + } catch (NoSuchJobException ex) { return 0; } } - private static boolean areParametersEquivalentExceptRunId (JobParameters parameters1, JobParameters parameters2) { - Map firstJobParameterMap = parameters1.getParameters(); - Map secondJobParameterMap = parameters2.getParameters(); + private static boolean areParametersEquivalentExceptRunId(JobParameters parameters1, JobParameters parameters2) { + Map> firstJobParameterMap = new HashMap<>(parameters1.getParameters()); + Map> secondJobParameterMap = new HashMap<>(parameters2.getParameters()); if (parameters1.getParameters().containsKey(RUN_ID_PARAMETER_NAME)) { firstJobParameterMap.remove(RUN_ID_PARAMETER_NAME); } @@ -87,8 +87,7 @@ public static void markPreviousJobAsFailed(String jobName, JobRepository jobRepo logger.info("Force restartPreviousExecution of job '" + jobName + "' with parameters: " + jobParameters); try { JobStatusManager.markLastJobAsFailed(jobRepository, jobName, jobParameters); - } - catch (NoPreviousJobExecutionException ex) { + } catch (NoPreviousJobExecutionException ex) { //Be forgiving if a previous job does not exist: just print a warning message logger.warn(ex.getMessage()); } diff --git a/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/service/nonhuman/dbsnp/DbsnpSubmittedVariantInactiveService.java b/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/service/nonhuman/dbsnp/DbsnpSubmittedVariantInactiveService.java index 3bc3f6d80..2a3265e5b 100644 --- a/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/service/nonhuman/dbsnp/DbsnpSubmittedVariantInactiveService.java +++ b/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/service/nonhuman/dbsnp/DbsnpSubmittedVariantInactiveService.java @@ -17,9 +17,8 @@ */ package uk.ac.ebi.eva.accession.core.service.nonhuman.dbsnp; -import uk.ac.ebi.ampt2d.commons.accession.persistence.repositories.IHistoryRepository; import uk.ac.ebi.ampt2d.commons.accession.persistence.mongodb.service.BasicMongoDbInactiveAccessionService; - +import uk.ac.ebi.ampt2d.commons.accession.persistence.repositories.IHistoryRepository; import uk.ac.ebi.eva.accession.core.model.ISubmittedVariant; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpSubmittedVariantEntity; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpSubmittedVariantInactiveEntity; diff --git a/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/service/nonhuman/eva/SubmittedVariantInactiveService.java b/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/service/nonhuman/eva/SubmittedVariantInactiveService.java index f04cbade5..9505c27be 100644 --- a/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/service/nonhuman/eva/SubmittedVariantInactiveService.java +++ b/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/service/nonhuman/eva/SubmittedVariantInactiveService.java @@ -17,9 +17,8 @@ */ package uk.ac.ebi.eva.accession.core.service.nonhuman.eva; -import uk.ac.ebi.ampt2d.commons.accession.persistence.repositories.IHistoryRepository; import uk.ac.ebi.ampt2d.commons.accession.persistence.mongodb.service.BasicMongoDbInactiveAccessionService; - +import uk.ac.ebi.ampt2d.commons.accession.persistence.repositories.IHistoryRepository; import uk.ac.ebi.eva.accession.core.model.ISubmittedVariant; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantInactiveEntity; diff --git a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/EVAObjectModelUtilsTest.java b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/EVAObjectModelUtilsTest.java index 4f609a14c..da09a904c 100644 --- a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/EVAObjectModelUtilsTest.java +++ b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/EVAObjectModelUtilsTest.java @@ -15,15 +15,18 @@ */ package uk.ac.ebi.eva.accession.core; -import org.junit.Test; +import org.junit.jupiter.api.Test; import uk.ac.ebi.eva.accession.core.model.ClusteredVariant; import uk.ac.ebi.eva.accession.core.model.SubmittedVariant; import uk.ac.ebi.eva.accession.core.model.eva.ClusteredVariantEntity; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; import uk.ac.ebi.eva.commons.core.models.VariantType; -import static org.junit.Assert.assertEquals; -import static uk.ac.ebi.eva.accession.core.EVAObjectModelUtils.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static uk.ac.ebi.eva.accession.core.EVAObjectModelUtils.getClusteredVariantHash; +import static uk.ac.ebi.eva.accession.core.EVAObjectModelUtils.toClusteredVariant; +import static uk.ac.ebi.eva.accession.core.EVAObjectModelUtils.toClusteredVariantEntity; +import static uk.ac.ebi.eva.accession.core.EVAObjectModelUtils.toSubmittedVariantEntity; public class EVAObjectModelUtilsTest { diff --git a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/batch/io/AccessionedVcfFactoryTest.java b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/batch/io/AccessionedVcfFactoryTest.java index e479d4061..4422c9caf 100644 --- a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/batch/io/AccessionedVcfFactoryTest.java +++ b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/batch/io/AccessionedVcfFactoryTest.java @@ -1,15 +1,20 @@ package uk.ac.ebi.eva.accession.core.batch.io; -import org.junit.BeforeClass; -import org.junit.Test; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; import uk.ac.ebi.eva.commons.core.models.factories.VariantVcfFactory; import uk.ac.ebi.eva.commons.core.models.pipeline.Variant; -import java.util.*; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; public class AccessionedVcfFactoryTest { @@ -20,7 +25,7 @@ public class AccessionedVcfFactoryTest { private static VariantVcfFactory factory; - @BeforeClass + @BeforeAll public static void setupClass() { factory = new AccessionedVcfFactory(); } diff --git a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/batch/io/ClusteredVariantDeprecationWriterTest.java b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/batch/io/ClusteredVariantDeprecationWriterTest.java index 224bd39dd..0e864cedb 100644 --- a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/batch/io/ClusteredVariantDeprecationWriterTest.java +++ b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/batch/io/ClusteredVariantDeprecationWriterTest.java @@ -16,23 +16,18 @@ package uk.ac.ebi.eva.accession.core.batch.io; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; -import com.mongodb.MongoClient; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.batch.item.Chunk; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; -import org.springframework.context.ApplicationContext; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.ampt2d.commons.accession.hashing.SHA1HashingFunction; - import uk.ac.ebi.eva.accession.core.configuration.nonhuman.ClusteredVariantAccessioningConfiguration; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.MongoConfiguration; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.SubmittedVariantAccessioningConfiguration; @@ -47,24 +42,21 @@ import uk.ac.ebi.eva.accession.core.service.nonhuman.SubmittedVariantAccessioningService; import uk.ac.ebi.eva.accession.core.summary.ClusteredVariantSummaryFunction; import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; -import uk.ac.ebi.eva.accession.core.test.rule.FixSpringMongoDbRule; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.commons.core.models.pipeline.Variant; import java.util.Arrays; import java.util.function.Function; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; -@RunWith(SpringRunner.class) +@ExtendWith(SpringExtension.class) @TestPropertySource("classpath:ss-deprecation-test.properties") @EnableAutoConfiguration @ContextConfiguration(classes = {MongoConfiguration.class, MongoTestConfiguration.class, SubmittedVariantAccessioningConfiguration.class, ClusteredVariantAccessioningConfiguration.class}) -public class ClusteredVariantDeprecationWriterTest { - - private static final String TEST_DB = "sve-deprecation-test"; - +public class ClusteredVariantDeprecationWriterTest extends MongoTestContainerHelper { private static final String ASSEMBLY = "GCA_000000001.1"; private static final int TAXONOMY = 60711; @@ -72,9 +64,6 @@ public class ClusteredVariantDeprecationWriterTest { @Autowired private Long accessioningMonotonicInitRs; - @Autowired - private MongoClient mongoClient; - @Autowired private MongoTemplate mongoTemplate; @@ -84,26 +73,14 @@ public class ClusteredVariantDeprecationWriterTest { private SubmittedVariantEntity ss1, ss2, ss3, ss4; private ClusteredVariantEntity rs1, rs2; - //Required by nosql-unit - @Autowired - private ApplicationContext applicationContext; - - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName(TEST_DB).build()); - - private void cleanup() { - mongoClient.dropDatabase(TEST_DB); - } - - @Before + @BeforeEach public void setUp() { - cleanup(); + mongoTemplate.getDb().drop(); } - @After + @AfterEach public void tearDown() { - cleanup(); + mongoTemplate.getDb().drop(); } @Test @@ -125,18 +102,18 @@ public void testDeprecateRS() { ClusteredVariantDeprecationWriter cveDeprecationWriter = new ClusteredVariantDeprecationWriter(ASSEMBLY, this.mongoTemplate, - this.submittedVariantAccessioningService, - this.accessioningMonotonicInitRs, - "TEST", "Deprecation test"); + this.submittedVariantAccessioningService, + this.accessioningMonotonicInitRs, + "TEST", "Deprecation test"); this.mongoTemplate.remove(ss1, this.mongoTemplate.getCollectionName(DbsnpSubmittedVariantEntity.class)); this.mongoTemplate.remove(ss2, this.mongoTemplate.getCollectionName(DbsnpSubmittedVariantEntity.class)); this.mongoTemplate.remove(ss3, this.mongoTemplate.getCollectionName(SubmittedVariantEntity.class)); - cveDeprecationWriter.write(Arrays.asList(rs1, rs2)); + cveDeprecationWriter.write(Chunk.of(rs1, rs2)); assertPostDeprecationDatabaseState(); // Ensure that the second run of deprecation does not do any harm - cveDeprecationWriter.write(Arrays.asList(rs1, rs2)); + cveDeprecationWriter.write(Chunk.of(rs1, rs2)); assertPostDeprecationDatabaseState(); } @@ -149,7 +126,7 @@ private void assertPostDeprecationDatabaseState() { assertEquals(0, this.mongoTemplate.findAll(ClusteredVariantOperationEntity.class).size()); DbsnpClusteredVariantOperationEntity rs1DeprecationOp = this.mongoTemplate.findById("RS_DEPRECATED_TEST_" + rs1.getHashedMessage(), - DbsnpClusteredVariantOperationEntity.class); + DbsnpClusteredVariantOperationEntity.class); assertNotNull(rs1DeprecationOp); assertEquals(rs1, rs1DeprecationOp.getInactiveObjects().get(0).toClusteredVariantEntity()); } @@ -158,20 +135,20 @@ private SubmittedVariantEntity createSS(Long ssAccession, Long rsAccession, Long String alternate) { return new SubmittedVariantEntity(ssAccession, "hash" + ssAccession, ASSEMBLY, TAXONOMY, - "PRJ1", "chr1", start, reference, alternate, rsAccession, false, false, false, - false, 1); + "PRJ1", "chr1", start, reference, alternate, rsAccession, false, false, false, + false, 1); } private ClusteredVariantEntity createRS(SubmittedVariantEntity sve) { - Function hashingFunction = new ClusteredVariantSummaryFunction().andThen( + Function hashingFunction = new ClusteredVariantSummaryFunction().andThen( new SHA1HashingFunction()); ClusteredVariant cv = new ClusteredVariant(sve.getReferenceSequenceAccession(), sve.getTaxonomyAccession(), - sve.getContig(), - sve.getStart(), - new Variant(sve.getContig(), sve.getStart(), sve.getStart(), - sve.getReferenceAllele(), - sve.getAlternateAllele()).getType(), - true, null); + sve.getContig(), + sve.getStart(), + new Variant(sve.getContig(), sve.getStart(), sve.getStart(), + sve.getReferenceAllele(), + sve.getAlternateAllele()).getType(), + true, null); String hash = hashingFunction.apply(cv); return new ClusteredVariantEntity(sve.getClusteredVariantAccession(), hash, cv); } diff --git a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/batch/io/DbsnpClusteredVariantWriterTest.java b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/batch/io/DbsnpClusteredVariantWriterTest.java index 86f4c5be1..35f2313a1 100644 --- a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/batch/io/DbsnpClusteredVariantWriterTest.java +++ b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/batch/io/DbsnpClusteredVariantWriterTest.java @@ -15,44 +15,45 @@ */ package uk.ac.ebi.eva.accession.core.batch.io; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.batch.item.Chunk; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.dao.DuplicateKeyException; +import org.springframework.data.mongodb.BulkOperationException; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.data.mongodb.core.query.Query; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.ampt2d.commons.accession.hashing.SHA1HashingFunction; - +import uk.ac.ebi.eva.accession.core.batch.listeners.ImportCounts; import uk.ac.ebi.eva.accession.core.model.ClusteredVariant; import uk.ac.ebi.eva.accession.core.model.IClusteredVariant; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpClusteredVariantEntity; import uk.ac.ebi.eva.accession.core.summary.ClusteredVariantSummaryFunction; -import uk.ac.ebi.eva.accession.core.batch.listeners.ImportCounts; import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import java.time.LocalDateTime; import java.time.Month; import java.util.Arrays; -import java.util.Collections; import java.util.List; import java.util.function.Function; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertThrows; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; import static uk.ac.ebi.eva.commons.core.models.VariantType.INDEL; import static uk.ac.ebi.eva.commons.core.models.VariantType.SNV; -@RunWith(SpringRunner.class) +@ExtendWith(SpringExtension.class) @TestPropertySource("classpath:test-variants-writer.properties") @ContextConfiguration(classes = {MongoTestConfiguration.class}) -public class DbsnpClusteredVariantWriterTest { +public class DbsnpClusteredVariantWriterTest extends MongoTestContainerHelper { private static final int TAXONOMY_1 = 3880; @@ -93,8 +94,9 @@ public class DbsnpClusteredVariantWriterTest { private ClusteredVariant clusteredVariant3; - @Before + @BeforeEach public void setUp() { + mongoTemplate.getDb().drop(); importCounts = new ImportCounts(); dbsnpClusteredVariantWriter = new DbsnpClusteredVariantWriter(mongoTemplate, importCounts); hashingFunction = new ClusteredVariantSummaryFunction().andThen(new SHA1HashingFunction()); @@ -103,8 +105,8 @@ public void setUp() { // variants and entity objects clusteredVariant1 = new ClusteredVariant(ASSEMBLY, TAXONOMY_1, CONTIG, START, SNV, true, CREATED_DATE); ClusteredVariant duplicateClusteredVariant1 = new ClusteredVariant(ASSEMBLY, TAXONOMY_1, CONTIG, START, SNV, - false, null); - clusteredVariant2 = new ClusteredVariant(ASSEMBLY, TAXONOMY_1, CONTIG, START+1, SNV, true, null); + false, null); + clusteredVariant2 = new ClusteredVariant(ASSEMBLY, TAXONOMY_1, CONTIG, START + 1, SNV, true, null); clusteredVariant3 = new ClusteredVariant(ASSEMBLY, TAXONOMY_1, CONTIG, START, INDEL, true, null); variantEntity1 = buildClusteredVariantEntity(ACCESSION_1, clusteredVariant1); duplicateVariantEntity1 = buildClusteredVariantEntity(ACCESSION_1, duplicateClusteredVariant1); @@ -112,37 +114,42 @@ public void setUp() { variantEntity3 = buildClusteredVariantEntity(ACCESSION_3, clusteredVariant3); } + @AfterEach + public void tearDown() { + mongoTemplate.getDb().drop(); + } + private DbsnpClusteredVariantEntity buildClusteredVariantEntity(long accession, ClusteredVariant clusteredVariant) { return new DbsnpClusteredVariantEntity(accession, hashingFunction.apply(clusteredVariant), clusteredVariant); } @Test public void saveSingleAccession() { - dbsnpClusteredVariantWriter.write(Collections.singletonList(variantEntity1)); + dbsnpClusteredVariantWriter.write(Chunk.of(variantEntity1)); assertJustOneVariantHasBeenStored(); } private void assertJustOneVariantHasBeenStored() { List storedVariants = mongoTemplate.find(new Query(), - DbsnpClusteredVariantEntity.class); + DbsnpClusteredVariantEntity.class); assertEquals(1, storedVariants.size()); assertTrue(storedVariants.contains(variantEntity1)); assertEquals(ACCESSION_1, storedVariants.get(0).getAccession()); assertEquals(clusteredVariant1, storedVariants.get(0).getModel()); assertEquals(1, importCounts.getClusteredVariantsWritten()); - assertEquals(CREATED_DATE, storedVariants.get(0).getModel().getCreatedDate()); } @Test public void exceptionThrownOnDuplicateIdenticalVariant() { - assertThrows(DuplicateKeyException.class, () -> - dbsnpClusteredVariantWriter.write(Arrays.asList(variantEntity1, variantEntity1))); + BulkOperationException exception = assertThrows(BulkOperationException.class, () -> + dbsnpClusteredVariantWriter.write(Chunk.of(variantEntity1, variantEntity1))); + assertTrue(exception.getMessage().contains("duplicate key error")); } @Test public void duplicateIdenticalVariantIsStoredJustOnce() { try { - dbsnpClusteredVariantWriter.write(Arrays.asList(variantEntity1, variantEntity1)); + dbsnpClusteredVariantWriter.write(Chunk.of(variantEntity1, variantEntity1)); fail(); } catch (Exception e) { // it's correct and expected that an exception is thrown here @@ -154,7 +161,7 @@ public void duplicateIdenticalVariantIsStoredJustOnce() { @Test public void duplicateNotIdenticalVariantIsStoredJustOnce() { try { - dbsnpClusteredVariantWriter.write(Arrays.asList(variantEntity1, duplicateVariantEntity1)); + dbsnpClusteredVariantWriter.write(Chunk.of(variantEntity1, duplicateVariantEntity1)); fail(); } catch (Exception e) { // it's correct and expected that an exception is thrown here @@ -164,13 +171,13 @@ public void duplicateNotIdenticalVariantIsStoredJustOnce() { @Test public void saveDifferentVariants() { - dbsnpClusteredVariantWriter.write(Arrays.asList(variantEntity1, variantEntity2, variantEntity3)); + dbsnpClusteredVariantWriter.write(Chunk.of(variantEntity1, variantEntity2, variantEntity3)); assertAllUniqueVariantsHaveBeenStored(); } private void assertAllUniqueVariantsHaveBeenStored() { List storedVariants = mongoTemplate.find(new Query(), - DbsnpClusteredVariantEntity.class); + DbsnpClusteredVariantEntity.class); assertEquals(3, storedVariants.size()); assertFalse(storedVariants.contains(duplicateVariantEntity1)); assertEquals(ACCESSION_1, storedVariants.get(0).getAccession()); @@ -185,10 +192,10 @@ private void assertAllUniqueVariantsHaveBeenStored() { @Test public void allNonDuplicatedRecordsWillBeWritten() { List batch = Arrays.asList(variantEntity1, variantEntity2, duplicateVariantEntity1, - variantEntity3); + variantEntity3); try { - dbsnpClusteredVariantWriter.write(batch); + dbsnpClusteredVariantWriter.write(new Chunk<>(batch)); fail(); } catch (Exception e) { // it's correct and expected that an exception is thrown here diff --git a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/batch/io/FastaSequenceReaderTest.java b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/batch/io/FastaSequenceReaderTest.java index 5fdfb50ee..fc748d7d3 100644 --- a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/batch/io/FastaSequenceReaderTest.java +++ b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/batch/io/FastaSequenceReaderTest.java @@ -16,13 +16,11 @@ package uk.ac.ebi.eva.accession.core.batch.io; import org.apache.commons.lang3.tuple.ImmutableTriple; -import org.junit.After; -import org.junit.Before; -import org.junit.Ignore; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; -import org.junit.rules.TemporaryFolder; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; +import uk.ac.ebi.eva.accession.core.utils.PipelineTemporaryFolderUtil; import java.io.File; import java.io.IOException; @@ -31,7 +29,8 @@ import java.nio.file.Path; import java.nio.file.Paths; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; public class FastaSequenceReaderTest { @@ -39,62 +38,54 @@ public class FastaSequenceReaderTest { private FastaSequenceReader reader; - @Rule - public final ExpectedException thrown = ExpectedException.none(); + public PipelineTemporaryFolderUtil temporaryFolderUtil = new PipelineTemporaryFolderUtil(); - @Rule - public TemporaryFolder temporaryFolder = new TemporaryFolder(); - - @Before + @BeforeEach public void setUp() throws Exception { reader = new FastaSequenceReader(Paths.get( FastaSequenceReaderTest.class.getResource("/input-files/fasta/Gallus_gallus-5.0.test.fa").toURI())); } - @After + @AfterEach public void tearDown() throws Exception { reader.close(); } @Test - public void getFirstNucleotideOfContig() throws Exception { + public void getFirstNucleotideOfContig() { assertEquals("T", reader.getSequence("22", 1, 1)); } @Test - public void getLastNucleotideOfContig() throws Exception { + public void getLastNucleotideOfContig() { assertEquals("G", reader.getSequence("22", 4729743, 4729743)); } @Test - public void getSequence() throws Exception { + public void getSequence() { // this sequence is split between three lines in the FASTA file assertEquals("GTTTCAAGTGGTTGTGACCCCCGCTGCACAGTCAGTTGGGTTAGGGTTAGGGTTAGGGTCAGTCACAGTCAGTTGTCAGACTGGTGTTTA", - reader.getSequence("22", 59986, 60075)); + reader.getSequence("22", 59986, 60075)); } @Test - public void endMustBeGreaterOrEqualsThanStart() throws Exception { - thrown.expect(IllegalArgumentException.class); - reader.getSequence("22", 1000, 999); + public void endMustBeGreaterOrEqualsThanStart() { + assertThrows(IllegalArgumentException.class, () -> reader.getSequence("22", 1000, 999)); } @Test - public void onlyPositiveCoordinatesAreAllowed() throws Exception { - thrown.expect(IllegalArgumentException.class); - reader.getSequence("22", -1, 5); + public void onlyPositiveCoordinatesAreAllowed() { + assertThrows(IllegalArgumentException.class, () -> reader.getSequence("22", -1, 5)); } @Test - public void coordinatesGreaterThanEndOfChromosomeAreNotAllowed() throws Exception { - thrown.expect(IllegalArgumentException.class); - reader.getSequence("22", 4729740, 4729750); + public void coordinatesGreaterThanEndOfChromosomeAreNotAllowed() { + assertThrows(IllegalArgumentException.class, () -> reader.getSequence("22", 4729740, 4729750)); } @Test - public void notExistentChromosome() throws Exception { - thrown.expect(IllegalArgumentException.class); - reader.getSequence("23", 1, 1); + public void notExistentChromosome() { + assertThrows(IllegalArgumentException.class, () -> reader.getSequence("23", 1, 1)); } @Test @@ -103,7 +94,7 @@ public void fastaWithNoDictionary() throws Exception { FastaSequenceReaderTest.class.getResource("/input-files/fasta/fastaWithNoDictionary.fa").toURI())); // this sequence is split between three lines in the FASTA file assertEquals("CAGCCGCAGTCCGGACAGCGCATGCGCCAGCCGCGAGACCGCACAGCGCATGCGCCAGCGCGAGTGACAGCG", - fastaSequenceReader.getSequence("22", 174, 245)); + fastaSequenceReader.getSequence("22", 174, 245)); } @Test @@ -113,11 +104,11 @@ public void fastaWithNoIndex() throws Exception { // this sequence is split between three lines in the FASTA file assertEquals("CAGCCGCAGTCCGGACAGCGCATGCGCCAGCCGCGAGACCGCACAGCGCATGCGCCAGCGCGAGTGACAGCG", - fastaSequenceReader.getSequence("22", 174, 245)); + fastaSequenceReader.getSequence("22", 174, 245)); } private FastaSequenceReader getFastaSequenceReader(String fastaFilename) throws IOException, URISyntaxException { - File temporaryFolderRoot = temporaryFolder.getRoot(); + File temporaryFolderRoot = temporaryFolderUtil.getRoot(); Path fasta = Files.copy( Paths.get(FastaSequenceReaderTest.class.getResource("/input-files/fasta/" + fastaFilename).toURI()), temporaryFolderRoot.toPath().resolve(fastaFilename)); @@ -132,7 +123,7 @@ private FastaSequenceReader getFastaSequenceReader(String fastaFilename) throws * forbids compressed fastas. You can comment the requirement by hand and run this test to see if it still applies. */ @Test - @Ignore + @Disabled public void htsDoesNotSupportCompressedFastas() throws URISyntaxException, IOException { String fastaFilename = "compressed.fa.gz"; FastaSequenceReader fastaSequenceReader = getFastaSequenceReader(fastaFilename); @@ -142,18 +133,17 @@ public void htsDoesNotSupportCompressedFastas() throws URISyntaxException, IOExc /** * For the rationale of this test, look at {@link #htsDoesNotSupportCompressedFastas()} and - * {@link FastaSequenceReader#checkFastaIsUncompressed(java.nio.file.Path)} + * {@link FastaSequenceReader#checkFastaIsUncompressed(java.nio.file.Path)} */ @Test public void shouldThrowOnCompressedFasta() throws URISyntaxException, IOException { String fastaFilename = "compressed.fa.gz"; - File temporaryFolderRoot = temporaryFolder.getRoot(); + File temporaryFolderRoot = temporaryFolderUtil.getRoot(); Path fasta = Files.copy( Paths.get(FastaSequenceReaderTest.class.getResource("/input-files/fasta/" + fastaFilename).toURI()), temporaryFolderRoot.toPath().resolve(fastaFilename)); - thrown.expect(IllegalArgumentException.class); - new FastaSequenceReader(fasta); + assertThrows(IllegalArgumentException.class, () -> new FastaSequenceReader(fasta)); } @Test diff --git a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/batch/io/SubmittedVariantDeprecationWriterTest.java b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/batch/io/SubmittedVariantDeprecationWriterTest.java index dcbd7f824..8179d4a33 100644 --- a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/batch/io/SubmittedVariantDeprecationWriterTest.java +++ b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/batch/io/SubmittedVariantDeprecationWriterTest.java @@ -16,23 +16,18 @@ package uk.ac.ebi.eva.accession.core.batch.io; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; -import com.mongodb.MongoClient; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.batch.item.Chunk; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; -import org.springframework.context.ApplicationContext; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.ampt2d.commons.accession.hashing.SHA1HashingFunction; - import uk.ac.ebi.eva.accession.core.configuration.nonhuman.ClusteredVariantAccessioningConfiguration; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.MongoConfiguration; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.SubmittedVariantAccessioningConfiguration; @@ -50,21 +45,21 @@ import uk.ac.ebi.eva.accession.core.service.nonhuman.SubmittedVariantAccessioningService; import uk.ac.ebi.eva.accession.core.summary.ClusteredVariantSummaryFunction; import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; -import uk.ac.ebi.eva.accession.core.test.rule.FixSpringMongoDbRule; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.commons.core.models.pipeline.Variant; import java.util.Arrays; import java.util.function.Function; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; -@RunWith(SpringRunner.class) +@ExtendWith(SpringExtension.class) @TestPropertySource("classpath:ss-deprecation-test.properties") @EnableAutoConfiguration @ContextConfiguration(classes = {MongoConfiguration.class, MongoTestConfiguration.class, SubmittedVariantAccessioningConfiguration.class, ClusteredVariantAccessioningConfiguration.class}) -public class SubmittedVariantDeprecationWriterTest { +public class SubmittedVariantDeprecationWriterTest extends MongoTestContainerHelper { private static final String TEST_DB = "sve-deprecation-test"; @@ -80,9 +75,6 @@ public class SubmittedVariantDeprecationWriterTest { @Autowired private Long accessioningMonotonicInitRs; - @Autowired - private MongoClient mongoClient; - @Autowired private MongoTemplate mongoTemplate; @@ -95,26 +87,14 @@ public class SubmittedVariantDeprecationWriterTest { private SubmittedVariantEntity ss1, ss2, ss3, ss4; private ClusteredVariantEntity rs1, rs1AnotherHash, rs2; - //Required by nosql-unit - @Autowired - private ApplicationContext applicationContext; - - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName(TEST_DB).build()); - - private void cleanup() { - mongoClient.dropDatabase(TEST_DB); - } - - @Before + @BeforeEach public void setUp() { - cleanup(); + mongoTemplate.getDb().drop(); } - @After + @AfterEach public void tearDown() { - cleanup(); + mongoTemplate.getDb().drop(); } @Test @@ -136,16 +116,16 @@ public void testDeprecateSS() { SubmittedVariantDeprecationWriter sveDeprecationWriter = new SubmittedVariantDeprecationWriter(ASSEMBLY, this.mongoTemplate, - this.submittedVariantAccessioningService, - this.clusteredVariantAccessioningService, - this.accessioningMonotonicInitSs, - this.accessioningMonotonicInitRs, - "TEST", REASON); - sveDeprecationWriter.write(Arrays.asList(ss1, ss2, ss3)); + this.submittedVariantAccessioningService, + this.clusteredVariantAccessioningService, + this.accessioningMonotonicInitSs, + this.accessioningMonotonicInitRs, + "TEST", REASON); + sveDeprecationWriter.write(Chunk.of(ss1, ss2, ss3)); assertPostDeprecationDatabaseState(); // Ensure that the second run of deprecation does not do any harm - sveDeprecationWriter.write(Arrays.asList(ss1, ss2, ss3)); + sveDeprecationWriter.write(Chunk.of(ss1, ss2, ss3)); assertPostDeprecationDatabaseState(); } @@ -180,7 +160,7 @@ private void assertPostDeprecationDatabaseState() { assertEquals(0, this.mongoTemplate.findAll(ClusteredVariantOperationEntity.class).size()); DbsnpClusteredVariantOperationEntity rs1DeprecationOp = this.mongoTemplate.findById("RS_DEPRECATED_TEST_" + rs1.getHashedMessage(), - DbsnpClusteredVariantOperationEntity.class); + DbsnpClusteredVariantOperationEntity.class); assertNotNull(rs1DeprecationOp); assertEquals(rs1, rs1DeprecationOp.getInactiveObjects().get(0).toClusteredVariantEntity()); } @@ -189,20 +169,20 @@ private SubmittedVariantEntity createSS(Long ssAccession, Long rsAccession, Long String alternate) { return new SubmittedVariantEntity(ssAccession, "hash" + ssAccession, ASSEMBLY, TAXONOMY, - "PRJ1", "chr1", start, reference, alternate, rsAccession, false, false, false, - false, 1); + "PRJ1", "chr1", start, reference, alternate, rsAccession, false, false, false, + false, 1); } private ClusteredVariantEntity createRS(SubmittedVariantEntity sve) { - Function hashingFunction = new ClusteredVariantSummaryFunction().andThen( + Function hashingFunction = new ClusteredVariantSummaryFunction().andThen( new SHA1HashingFunction()); ClusteredVariant cv = new ClusteredVariant(sve.getReferenceSequenceAccession(), sve.getTaxonomyAccession(), - sve.getContig(), - sve.getStart(), - new Variant(sve.getContig(), sve.getStart(), sve.getStart(), - sve.getReferenceAllele(), - sve.getAlternateAllele()).getType(), - true, null); + sve.getContig(), + sve.getStart(), + new Variant(sve.getContig(), sve.getStart(), sve.getStart(), + sve.getReferenceAllele(), + sve.getAlternateAllele()).getType(), + true, null); String hash = hashingFunction.apply(cv); return new ClusteredVariantEntity(sve.getClusteredVariantAccession(), hash, cv); } @@ -235,11 +215,11 @@ public void testDeprecateSSWithSpecificRSHash() { this.accessioningMonotonicInitSs, this.accessioningMonotonicInitRs, "TEST", REASON); - sveDeprecationWriter.write(Arrays.asList(ss1, ss2, ss3)); + sveDeprecationWriter.write(Chunk.of(ss1, ss2, ss3)); assertPostDeprecationDatabaseStateForSpecificRSHash(); // Ensure that the second run of deprecation does not do any harm - sveDeprecationWriter.write(Arrays.asList(ss1, ss2, ss3)); + sveDeprecationWriter.write(Chunk.of(ss1, ss2, ss3)); assertPostDeprecationDatabaseStateForSpecificRSHash(); } diff --git a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/batch/policies/IllegalStartSkipPolicyTest.java b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/batch/policies/IllegalStartSkipPolicyTest.java index 34e4b7dca..186989b86 100644 --- a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/batch/policies/IllegalStartSkipPolicyTest.java +++ b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/batch/policies/IllegalStartSkipPolicyTest.java @@ -15,19 +15,19 @@ */ package uk.ac.ebi.eva.accession.core.batch.policies; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import uk.ac.ebi.eva.accession.core.exceptions.PositionOutsideOfContigException; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; public class IllegalStartSkipPolicyTest { private IllegalStartSkipPolicy illegalStartSkipPolicy; - @Before + @BeforeEach public void setUp() throws Exception { illegalStartSkipPolicy = new IllegalStartSkipPolicy(); } diff --git a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/contig/ContigMappingTest.java b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/contig/ContigMappingTest.java index 45ab7832f..4061ea353 100644 --- a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/contig/ContigMappingTest.java +++ b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/contig/ContigMappingTest.java @@ -15,13 +15,13 @@ */ package uk.ac.ebi.eva.accession.core.contig; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; public class ContigMappingTest { @@ -71,7 +71,7 @@ public class ContigMappingTest { private static final int MISSING_UCSC_ROWS = 1; - @Before + @BeforeEach public void setUp() throws Exception { String fileString = ContigMappingTest.class.getResource( "/input-files/assembly-report/GCA_000001635.8_Mus_musculus-grcm38.p6_assembly_report.txt").toString(); diff --git a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/model/eva/ClusteredVariantEntityIntegrationTest.java b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/model/eva/ClusteredVariantEntityIntegrationTest.java index 6ddf397f7..7d535412e 100644 --- a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/model/eva/ClusteredVariantEntityIntegrationTest.java +++ b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/model/eva/ClusteredVariantEntityIntegrationTest.java @@ -15,40 +15,36 @@ */ package uk.ac.ebi.eva.accession.core.model.eva; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; -import org.springframework.context.ApplicationContext; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.data.mongodb.core.query.Query; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.ampt2d.commons.accession.hashing.SHA1HashingFunction; - import uk.ac.ebi.eva.accession.core.configuration.nonhuman.MongoConfiguration; import uk.ac.ebi.eva.accession.core.model.ClusteredVariant; import uk.ac.ebi.eva.accession.core.model.IClusteredVariant; import uk.ac.ebi.eva.accession.core.summary.ClusteredVariantSummaryFunction; -import uk.ac.ebi.eva.accession.core.test.rule.FixSpringMongoDbRule; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.commons.core.models.VariantType; import java.util.Collections; import java.util.function.Function; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; -@RunWith(SpringRunner.class) +@ExtendWith(SpringExtension.class) @EnableAutoConfiguration -@ContextConfiguration(classes = {MongoConfiguration.class}) +@ContextConfiguration(classes = {MongoConfiguration.class, MongoTestConfiguration.class}) @TestPropertySource("classpath:test-model.properties") -public class ClusteredVariantEntityIntegrationTest { +public class ClusteredVariantEntityIntegrationTest extends MongoTestContainerHelper { private static final String TEST_DB = "test-db"; @@ -57,20 +53,13 @@ public class ClusteredVariantEntityIntegrationTest { @Autowired private MongoTemplate mongoTemplate; - //Required by nosql-unit - @Autowired - private ApplicationContext applicationContext; - - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName(TEST_DB).build()); - - @Before + @BeforeEach public void setUp() { + mongoTemplate.getDb().drop(); clusteredHashingFunction = new ClusteredVariantSummaryFunction().andThen(new SHA1HashingFunction()); } - @After + @AfterEach public void tearDown() { mongoTemplate.getDb().drop(); } @@ -105,7 +94,7 @@ private ClusteredVariantEntity createClusteredVariantEntity(Integer mapWeight) { ClusteredVariant variant = createClusteredVariant(); String hash = clusteredHashingFunction.apply(variant); ClusteredVariantEntity variantEntity = new ClusteredVariantEntity(3000000000L, hash, "asm", 1000, "contig", 100, - VariantType.SNV, false, null, 1, mapWeight); + VariantType.SNV, false, null, 1, mapWeight); return variantEntity; } diff --git a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/model/eva/ReleaseRecordSubmittedVariantEntityTest.java b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/model/eva/ReleaseRecordSubmittedVariantEntityTest.java index 061ef0708..c4ff019bf 100644 --- a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/model/eva/ReleaseRecordSubmittedVariantEntityTest.java +++ b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/model/eva/ReleaseRecordSubmittedVariantEntityTest.java @@ -1,9 +1,9 @@ package uk.ac.ebi.eva.accession.core.model.eva; -import org.junit.Test; +import org.junit.jupiter.api.Test; import uk.ac.ebi.eva.accession.core.model.ReleaseRecordSubmittedVariantEntity; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; public class ReleaseRecordSubmittedVariantEntityTest { diff --git a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/model/eva/SubmittedVariantEntityIntegrationTest.java b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/model/eva/SubmittedVariantEntityIntegrationTest.java index d45113261..92ed34acd 100644 --- a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/model/eva/SubmittedVariantEntityIntegrationTest.java +++ b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/model/eva/SubmittedVariantEntityIntegrationTest.java @@ -15,61 +15,47 @@ */ package uk.ac.ebi.eva.accession.core.model.eva; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; -import org.springframework.context.ApplicationContext; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.data.mongodb.core.query.Query; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.ampt2d.commons.accession.hashing.SHA1HashingFunction; - import uk.ac.ebi.eva.accession.core.configuration.nonhuman.MongoConfiguration; import uk.ac.ebi.eva.accession.core.model.ISubmittedVariant; import uk.ac.ebi.eva.accession.core.model.SubmittedVariant; import uk.ac.ebi.eva.accession.core.summary.SubmittedVariantSummaryFunction; -import uk.ac.ebi.eva.accession.core.test.rule.FixSpringMongoDbRule; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import java.util.Collections; import java.util.function.Function; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; -@RunWith(SpringRunner.class) +@ExtendWith(SpringExtension.class) @EnableAutoConfiguration -@ContextConfiguration(classes = {MongoConfiguration.class}) +@ContextConfiguration(classes = {MongoConfiguration.class, MongoTestConfiguration.class}) @TestPropertySource("classpath:test-model.properties") -public class SubmittedVariantEntityIntegrationTest { - - private static final String TEST_DB = "test-db"; - +public class SubmittedVariantEntityIntegrationTest extends MongoTestContainerHelper { private Function submittedHashingFunction; @Autowired private MongoTemplate mongoTemplate; - //Required by nosql-unit - @Autowired - private ApplicationContext applicationContext; - - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName(TEST_DB).build()); - - @Before + @BeforeEach public void setUp() { + mongoTemplate.getDb().drop(); submittedHashingFunction = new SubmittedVariantSummaryFunction().andThen(new SHA1HashingFunction()); } - @After + @AfterEach public void tearDown() { mongoTemplate.getDb().drop(); } @@ -104,8 +90,8 @@ private SubmittedVariantEntity createSubmittedVariantEntity(Integer mapWeight) { SubmittedVariant variant = createSubmittedVariant(); String hash = submittedHashingFunction.apply(variant); SubmittedVariantEntity variantEntity = new SubmittedVariantEntity(5000000000L, hash, "asm", 1000, "project", - "contig", 100, "A", "T", null, false, false, - false, false, 1, mapWeight); + "contig", 100, "A", "T", null, false, false, + false, false, 1, mapWeight); return variantEntity; } diff --git a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/model/eva/SubmittedVariantEntityTest.java b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/model/eva/SubmittedVariantEntityTest.java index 17ac67664..094d35a9f 100644 --- a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/model/eva/SubmittedVariantEntityTest.java +++ b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/model/eva/SubmittedVariantEntityTest.java @@ -15,15 +15,14 @@ */ package uk.ac.ebi.eva.accession.core.model.eva; -import org.junit.Test; - +import org.junit.jupiter.api.Test; import uk.ac.ebi.eva.accession.core.model.SubmittedVariant; -import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; import java.time.LocalDateTime; import java.time.Month; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; public class SubmittedVariantEntityTest { @@ -51,7 +50,7 @@ public class SubmittedVariantEntityTest { public void getModelWithFlagsTrue() { SubmittedVariant variant = new SubmittedVariant(ASSEMBLY_ACCESSION, TAXONOMY_ACCESSION, PROJECT_ACCESSION, CONTIG, START, - REF_A, ALT_T, CLUSTERED_VARIANT, true, true, true, true, null); + REF_A, ALT_T, CLUSTERED_VARIANT, true, true, true, true, null); SubmittedVariantEntity entity = new SubmittedVariantEntity(UNUSED_ACCESSION, UNUSED_HASHED_MESSAGE, variant, 1); assertEquals(variant, entity.getModel()); @@ -61,7 +60,7 @@ public void getModelWithFlagsTrue() { public void getModelWithFlagsFalse() { SubmittedVariant variant = new SubmittedVariant(ASSEMBLY_ACCESSION, TAXONOMY_ACCESSION, PROJECT_ACCESSION, CONTIG, START, - REF_A, ALT_T, CLUSTERED_VARIANT, false, false, false, false, null); + REF_A, ALT_T, CLUSTERED_VARIANT, false, false, false, false, null); SubmittedVariantEntity entity = new SubmittedVariantEntity(UNUSED_ACCESSION, UNUSED_HASHED_MESSAGE, variant, 1); assertEquals(variant, entity.getModel()); @@ -71,7 +70,7 @@ public void getModelWithFlagsFalse() { public void getModelWithDefaultFlags() { SubmittedVariant variant = new SubmittedVariant(ASSEMBLY_ACCESSION, TAXONOMY_ACCESSION, PROJECT_ACCESSION, CONTIG, START, - REF_A, ALT_T, CLUSTERED_VARIANT); + REF_A, ALT_T, CLUSTERED_VARIANT); SubmittedVariantEntity entity = new SubmittedVariantEntity(UNUSED_ACCESSION, UNUSED_HASHED_MESSAGE, variant, 1); variant.setSupportedByEvidence(true); @@ -81,11 +80,11 @@ public void getModelWithDefaultFlags() { assertEquals(variant, entity.getModel()); } - @Test(expected = IllegalArgumentException.class) + @Test public void flagsShouldNotBeNull() { - new SubmittedVariantEntity(UNUSED_ACCESSION, UNUSED_HASHED_MESSAGE, ASSEMBLY_ACCESSION, TAXONOMY_ACCESSION, - PROJECT_ACCESSION, CONTIG, START, REF_A, ALT_T, CLUSTERED_VARIANT, null, null, null, - null, 1); + assertThrows(IllegalArgumentException.class, () -> new SubmittedVariantEntity(UNUSED_ACCESSION, UNUSED_HASHED_MESSAGE, ASSEMBLY_ACCESSION, TAXONOMY_ACCESSION, + PROJECT_ACCESSION, CONTIG, START, REF_A, ALT_T, CLUSTERED_VARIANT, null, null, null, + null, 1)); } @Test @@ -94,7 +93,7 @@ public void creationDateIsReturnedInTheModel() { LocalDateTime createdDate = LocalDateTime.of(2018, Month.SEPTEMBER, 18, 9, 0); SubmittedVariant variant = new SubmittedVariant(ASSEMBLY_ACCESSION, TAXONOMY_ACCESSION, PROJECT_ACCESSION, CONTIG, START, - REF_A, ALT_T, CLUSTERED_VARIANT, false, false, false, false, createdDate); + REF_A, ALT_T, CLUSTERED_VARIANT, false, false, false, false, createdDate); SubmittedVariantEntity entity = new SubmittedVariantEntity(UNUSED_ACCESSION, UNUSED_HASHED_MESSAGE, variant, 1); assertEquals(createdDate, entity.getModel().getCreatedDate()); @@ -105,7 +104,7 @@ public void testChangeSVERefAltToUpperCase() { SubmittedVariant submittedVariant = new SubmittedVariant(ASSEMBLY_ACCESSION, TAXONOMY_ACCESSION, PROJECT_ACCESSION, CONTIG, START, "a", "t", CLUSTERED_VARIANT, false, false, false, false, LocalDateTime.of(2018, Month.SEPTEMBER, 18, 9, 0)); - SubmittedVariantEntity submittedVariantEntity = new SubmittedVariantEntity(123L, "hashedMessage", submittedVariant,1 ); + SubmittedVariantEntity submittedVariantEntity = new SubmittedVariantEntity(123L, "hashedMessage", submittedVariant, 1); assertEquals("A", submittedVariantEntity.getReferenceAllele()); assertEquals("T", submittedVariantEntity.getAlternateAllele()); } diff --git a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/mongoTemplate/MongoTemplateTest.java b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/mongoTemplate/MongoTemplateTest.java index 5c632e4ac..d90976752 100644 --- a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/mongoTemplate/MongoTemplateTest.java +++ b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/mongoTemplate/MongoTemplateTest.java @@ -16,33 +16,28 @@ package uk.ac.ebi.eva.accession.core.mongoTemplate; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.dao.DataAccessException; import org.springframework.data.mongodb.core.MongoTemplate; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.eva.commons.core.models.AnnotationMetadata; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.Mockito.doThrow; -@RunWith(SpringRunner.class) +@ExtendWith(SpringExtension.class) public class MongoTemplateTest { @MockBean private MongoTemplate mongoTemplate; - @Rule - public ExpectedException exceptionRule = ExpectedException.none(); - - @Test(expected = DataAccessException.class) + @Test public void testMongoTemplateWriteResultChecking() { AnnotationMetadata annotationMetadata = new AnnotationMetadata("vep_1", "vep_cache_1", true); - doThrow(new DataAccessException("Simulated exception") {}) - .when(mongoTemplate).save(annotationMetadata, "AnnotationMetadata"); + doThrow(new DataAccessException("Simulated exception") { + }).when(mongoTemplate).save(annotationMetadata, "AnnotationMetadata"); - mongoTemplate.save(annotationMetadata, "AnnotationMetadata"); + assertThrows(DataAccessException.class, () -> mongoTemplate.save(annotationMetadata, "AnnotationMetadata")); } } diff --git a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/repository/nonhuman/eva/ClusteredVariantAccessioningRepositoryTest.java b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/repository/nonhuman/eva/ClusteredVariantAccessioningRepositoryTest.java index 35e48013e..f4911b2e4 100644 --- a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/repository/nonhuman/eva/ClusteredVariantAccessioningRepositoryTest.java +++ b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/repository/nonhuman/eva/ClusteredVariantAccessioningRepositoryTest.java @@ -15,28 +15,23 @@ */ package uk.ac.ebi.eva.accession.core.repository.nonhuman.eva; -import com.lordofthejars.nosqlunit.annotation.UsingDataSet; -import com.lordofthejars.nosqlunit.core.LoadStrategyEnum; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.autoconfigure.orm.jpa.DataJpaTest; -import org.springframework.context.ApplicationContext; +import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.ampt2d.commons.accession.persistence.models.AccessionProjection; import uk.ac.ebi.ampt2d.commons.accession.persistence.mongodb.document.AccessionedDocument; - -import uk.ac.ebi.eva.accession.core.model.ClusteredVariant; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.ClusteredVariantAccessioningConfiguration; +import uk.ac.ebi.eva.accession.core.model.ClusteredVariant; import uk.ac.ebi.eva.accession.core.model.eva.ClusteredVariantEntity; import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; -import uk.ac.ebi.eva.accession.core.test.rule.FixSpringMongoDbRule; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.commons.core.models.VariantType; import java.time.LocalDateTime; @@ -47,13 +42,13 @@ import java.util.TreeSet; import java.util.stream.Collectors; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; -@RunWith(SpringRunner.class) +@ExtendWith(SpringExtension.class) @DataJpaTest @TestPropertySource("classpath:ss-accession-test.properties") @ContextConfiguration(classes = {ClusteredVariantAccessioningConfiguration.class, MongoTestConfiguration.class}) -public class ClusteredVariantAccessioningRepositoryTest { +public class ClusteredVariantAccessioningRepositoryTest extends MongoTestContainerHelper { private ClusteredVariant clusteredVariant; @@ -62,25 +57,26 @@ public class ClusteredVariantAccessioningRepositoryTest { @Autowired private ClusteredVariantAccessioningRepository repository; - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName("submitted-variants-test").build()); - - //Required by nosql-unit @Autowired - private ApplicationContext applicationContext; + private MongoTemplate mongoTemplate; - @Before + @BeforeEach public void setUp() { + mongoTemplate.getDb().drop(); clusteredVariant = new ClusteredVariant("GCA_000003055.3", 9913, "21", 20800319, VariantType.SNV, false, - LocalDateTime.of(2000, Month.SEPTEMBER, 19, 18, 2, 0)); + LocalDateTime.of(2000, Month.SEPTEMBER, 19, 18, 2, 0)); newClusteredVariant = new ClusteredVariant("assembly", 1111, "contig_2", 100, VariantType.SNV, false, - LocalDateTime.of(2000, Month.SEPTEMBER, 19, 18, 2, 0)); + LocalDateTime.of(2000, Month.SEPTEMBER, 19, 18, 2, 0)); } - @UsingDataSet(loadStrategy = LoadStrategyEnum.DELETE_ALL) + @AfterEach + void cleanDb() { + mongoTemplate.getDb().drop(); + } + + @Test public void queryAccessionRange() { long firstAccession = 1000L; @@ -92,25 +88,24 @@ public void queryAccessionRange() { repository.saveAll(variants); assertAccessionsEquals(Arrays.asList(firstAccession), - repository.findByAccessionGreaterThanEqualAndAccessionLessThanEqual(1000L, 1000L)); + repository.findByAccessionGreaterThanEqualAndAccessionLessThanEqual(1000L, 1000L)); assertAccessionsEquals(Arrays.asList(), - repository.findByAccessionGreaterThanEqualAndAccessionLessThanEqual(1001L, 1001L)); + repository.findByAccessionGreaterThanEqualAndAccessionLessThanEqual(1001L, 1001L)); assertAccessionsEquals(Arrays.asList(firstAccession, secondAccession), - repository.findByAccessionGreaterThanEqualAndAccessionLessThanEqual(1000L, 1005L)); + repository.findByAccessionGreaterThanEqualAndAccessionLessThanEqual(1000L, 1005L)); assertAccessionsEquals(Arrays.asList(secondAccession), - repository.findByAccessionGreaterThanEqualAndAccessionLessThanEqual(1001L, 1005L)); + repository.findByAccessionGreaterThanEqualAndAccessionLessThanEqual(1001L, 1005L)); } private void assertAccessionsEquals(List expectedAccessions, List> accessionsProjection) { assertEquals(new TreeSet<>(expectedAccessions), - accessionsProjection.stream().map(AccessionProjection::getAccession).collect(Collectors.toSet())); + accessionsProjection.stream().map(AccessionProjection::getAccession).collect(Collectors.toSet())); } - @UsingDataSet(loadStrategy = LoadStrategyEnum.DELETE_ALL) @Test public void testQueryAssemblyAndAccessionFilter() { long firstAccession = 1000L; @@ -121,12 +116,12 @@ public void testQueryAssemblyAndAccessionFilter() { repository.saveAll(variants); - List accessions = repository.findByAssemblyAccessionAndAccessionIn( + List accessions = repository.findByAssemblyAccessionAndAccessionIn( "assembly", Arrays.asList(firstAccession, secondAccession) ).stream().map(AccessionedDocument::getAccession).collect(Collectors.toList()); assertEquals(1, accessions.size()); - accessions = repository.findByAssemblyAccessionAndAccessionIn( + accessions = repository.findByAssemblyAccessionAndAccessionIn( "assembly", Collections.singletonList(firstAccession) ).stream().map(AccessionedDocument::getAccession).collect(Collectors.toList()); assertEquals(0, accessions.size()); diff --git a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/repository/nonhuman/eva/SubmittedVariantAccessioningRepositoryTest.java b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/repository/nonhuman/eva/SubmittedVariantAccessioningRepositoryTest.java index b27dd3d71..bb6d40b05 100644 --- a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/repository/nonhuman/eva/SubmittedVariantAccessioningRepositoryTest.java +++ b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/repository/nonhuman/eva/SubmittedVariantAccessioningRepositoryTest.java @@ -15,29 +15,23 @@ */ package uk.ac.ebi.eva.accession.core.repository.nonhuman.eva; -import com.lordofthejars.nosqlunit.annotation.UsingDataSet; -import com.lordofthejars.nosqlunit.core.LoadStrategyEnum; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.autoconfigure.orm.jpa.DataJpaTest; -import org.springframework.context.ApplicationContext; +import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.ampt2d.commons.accession.persistence.models.AccessionProjection; import uk.ac.ebi.ampt2d.commons.accession.persistence.mongodb.document.AccessionedDocument; - -import uk.ac.ebi.eva.accession.core.model.SubmittedVariant; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.SubmittedVariantAccessioningConfiguration; +import uk.ac.ebi.eva.accession.core.model.SubmittedVariant; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; +import uk.ac.ebi.eva.accession.core.test.configuration.JPATestConfiguration; import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; -import uk.ac.ebi.eva.accession.core.test.rule.FixSpringMongoDbRule; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import java.util.Arrays; import java.util.Collections; @@ -45,16 +39,16 @@ import java.util.TreeSet; import java.util.stream.Collectors; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import static uk.ac.ebi.eva.accession.core.model.ISubmittedVariant.DEFAULT_ASSEMBLY_MATCH; import static uk.ac.ebi.eva.accession.core.model.ISubmittedVariant.DEFAULT_SUPPORTED_BY_EVIDENCE; import static uk.ac.ebi.eva.accession.core.model.ISubmittedVariant.DEFAULT_VALIDATED; -@RunWith(SpringRunner.class) -@DataJpaTest +@ExtendWith(SpringExtension.class) @TestPropertySource("classpath:ss-accession-test.properties") -@ContextConfiguration(classes = {SubmittedVariantAccessioningConfiguration.class, MongoTestConfiguration.class}) -public class SubmittedVariantAccessioningRepositoryTest { +@ContextConfiguration(classes = {SubmittedVariantAccessioningConfiguration.class, MongoTestConfiguration.class, + JPATestConfiguration.class}) +public class SubmittedVariantAccessioningRepositoryTest extends MongoTestContainerHelper { private static final Long CLUSTERED_VARIANT = null; @@ -67,32 +61,30 @@ public class SubmittedVariantAccessioningRepositoryTest { @Autowired private SubmittedVariantAccessioningRepository repository; - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName("submitted-variants-test").build()); - - //Required by nosql-unit @Autowired - private ApplicationContext applicationContext; + private MongoTemplate mongoTemplate; - @Rule - public ExpectedException expectedException = ExpectedException.none(); - - @Before + @BeforeEach public void setUp() { + mongoTemplate.getDb().drop(); submittedVariant = new SubmittedVariant("GCA_000003055.3", 9913, PROJECT, "21", 20800319, "C", "T", - CLUSTERED_VARIANT, DEFAULT_SUPPORTED_BY_EVIDENCE, - DEFAULT_ASSEMBLY_MATCH, true, - DEFAULT_VALIDATED, null); + CLUSTERED_VARIANT, DEFAULT_SUPPORTED_BY_EVIDENCE, + DEFAULT_ASSEMBLY_MATCH, true, + DEFAULT_VALIDATED, null); newSubmittedVariant = new SubmittedVariant("assembly", 1111, "project", "contig_2", 100, "ref", "alt", - CLUSTERED_VARIANT, DEFAULT_SUPPORTED_BY_EVIDENCE, - DEFAULT_ASSEMBLY_MATCH, true, - DEFAULT_VALIDATED, null); + CLUSTERED_VARIANT, DEFAULT_SUPPORTED_BY_EVIDENCE, + DEFAULT_ASSEMBLY_MATCH, true, + DEFAULT_VALIDATED, null); + + } + @AfterEach + void cleanDb() { + mongoTemplate.getDb().drop(); } - @UsingDataSet(loadStrategy = LoadStrategyEnum.DELETE_ALL) + @Test public void queryAccessionRange() { long firstAccession = 1000L; @@ -104,25 +96,24 @@ public void queryAccessionRange() { repository.saveAll(variants); assertAccessionsEquals(Arrays.asList(firstAccession), - repository.findByAccessionGreaterThanEqualAndAccessionLessThanEqual(1000L, 1000L)); + repository.findByAccessionGreaterThanEqualAndAccessionLessThanEqual(1000L, 1000L)); assertAccessionsEquals(Arrays.asList(), - repository.findByAccessionGreaterThanEqualAndAccessionLessThanEqual(1001L, 1001L)); + repository.findByAccessionGreaterThanEqualAndAccessionLessThanEqual(1001L, 1001L)); assertAccessionsEquals(Arrays.asList(firstAccession, secondAccession), - repository.findByAccessionGreaterThanEqualAndAccessionLessThanEqual(1000L, 1005L)); + repository.findByAccessionGreaterThanEqualAndAccessionLessThanEqual(1000L, 1005L)); assertAccessionsEquals(Arrays.asList(secondAccession), - repository.findByAccessionGreaterThanEqualAndAccessionLessThanEqual(1001L, 1005L)); + repository.findByAccessionGreaterThanEqualAndAccessionLessThanEqual(1001L, 1005L)); } private void assertAccessionsEquals(List expectedAccessions, List> accessionsProjection) { assertEquals(new TreeSet<>(expectedAccessions), - accessionsProjection.stream().map(AccessionProjection::getAccession).collect(Collectors.toSet())); + accessionsProjection.stream().map(AccessionProjection::getAccession).collect(Collectors.toSet())); } - @UsingDataSet(loadStrategy = LoadStrategyEnum.DELETE_ALL) @Test public void testQueryAssemblyAndAccessionFilter() { long firstAccession = 1000L; @@ -132,12 +123,12 @@ public void testQueryAssemblyAndAccessionFilter() { new SubmittedVariantEntity(secondAccession, "hash-2", newSubmittedVariant, 1)); repository.saveAll(variants); - List accessions = repository.findByReferenceSequenceAccessionAndAccessionIn( + List accessions = repository.findByReferenceSequenceAccessionAndAccessionIn( "assembly", Arrays.asList(firstAccession, secondAccession) ).stream().map(AccessionedDocument::getAccession).collect(Collectors.toList()); assertEquals(1, accessions.size()); - accessions = repository.findByReferenceSequenceAccessionAndAccessionIn( + accessions = repository.findByReferenceSequenceAccessionAndAccessionIn( "assembly", Collections.singletonList(firstAccession) ).stream().map(AccessionedDocument::getAccession).collect(Collectors.toList()); assertEquals(0, accessions.size()); diff --git a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/service/human/dbsnp/HumanDbsnpClusteredVariantAccessioningServiceTest.java b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/service/human/dbsnp/HumanDbsnpClusteredVariantAccessioningServiceTest.java index 642f2f151..30682e090 100644 --- a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/service/human/dbsnp/HumanDbsnpClusteredVariantAccessioningServiceTest.java +++ b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/service/human/dbsnp/HumanDbsnpClusteredVariantAccessioningServiceTest.java @@ -15,26 +15,25 @@ */ package uk.ac.ebi.eva.accession.core.service.human.dbsnp; -import com.lordofthejars.nosqlunit.annotation.UsingDataSet; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; -import org.junit.Ignore; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.boot.test.autoconfigure.orm.jpa.DataJpaTest; -import org.springframework.context.ApplicationContext; +import org.springframework.core.io.ResourceLoader; +import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.ampt2d.commons.accession.core.models.AccessionWrapper; - import uk.ac.ebi.eva.accession.core.model.ClusteredVariant; import uk.ac.ebi.eva.accession.core.model.IClusteredVariant; +import uk.ac.ebi.eva.accession.core.test.configuration.JPATestConfiguration; import uk.ac.ebi.eva.accession.core.test.configuration.human.MongoHumanTestConfiguration; -import uk.ac.ebi.eva.accession.core.test.rule.FixSpringMongoDbRule; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; +import uk.ac.ebi.eva.accession.core.utils.MongoTestDataLoader; import uk.ac.ebi.eva.commons.core.models.VariantType; import java.time.LocalDateTime; @@ -42,14 +41,12 @@ import java.util.Collections; import java.util.List; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; -@RunWith(SpringRunner.class) -@DataJpaTest +@ExtendWith(SpringExtension.class) @TestPropertySource("classpath:ss-accession-test.properties") -@UsingDataSet(locations = {"/test-data/dbsnpClusteredVariantEntity.json", "/test-data/dbsnpClusteredVariantOperationEntity.json"}) -@ContextConfiguration(classes = {MongoHumanTestConfiguration.class}) -public class HumanDbsnpClusteredVariantAccessioningServiceTest { +@ContextConfiguration(classes = {MongoHumanTestConfiguration.class, JPATestConfiguration.class}) +public class HumanDbsnpClusteredVariantAccessioningServiceTest extends MongoTestContainerHelper { private static final Long HUMAN_ACTIVE_RS_ID_1 = 1118L; @@ -64,16 +61,27 @@ public class HumanDbsnpClusteredVariantAccessioningServiceTest { "CM000663.2", 72348112L, VariantType.INDEL, false, LocalDateTime.of(2017, Month.NOVEMBER, 11, 9, 55, 0)); @Autowired - @Qualifier("humanService") - private HumanDbsnpClusteredVariantAccessioningService humanService; + private MongoTemplate mongoTemplate; - //Required by nosql-unit @Autowired - private ApplicationContext applicationContext; + private ResourceLoader resourceLoader; + + @BeforeEach + public void setUp() { + mongoTemplate.getDb().drop(); + MongoTestDataLoader mongoTestDataLoader = new MongoTestDataLoader(mongoTemplate, resourceLoader); + mongoTestDataLoader.load("/test-data/dbsnpClusteredVariantEntity.json"); + mongoTestDataLoader.load("/test-data/dbsnpClusteredVariantOperationEntity.json"); + } - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName("human-variants-test").build()); + @AfterEach + public void tearDown() { + mongoTemplate.getDb().drop(); + } + + @Autowired + @Qualifier("humanService") + private HumanDbsnpClusteredVariantAccessioningService humanService; @Test public void getHumanActiveVariant() { @@ -85,7 +93,7 @@ public void getHumanActiveVariant() { } @Test - @Ignore("humanService.getAllByAccession is not returning multiple variants yet") + @Disabled("humanService.getAllByAccession is not returning multiple variants yet") public void getHumanActiveMultipleVariants() { List> clusteredVariants = humanService.getAllByAccession(HUMAN_ACTIVE_RS_ID_2); diff --git a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/service/nonhuman/ClusteredVariantAccessioningServiceTest.java b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/service/nonhuman/ClusteredVariantAccessioningServiceTest.java index 637596ebb..ed9324c80 100644 --- a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/service/nonhuman/ClusteredVariantAccessioningServiceTest.java +++ b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/service/nonhuman/ClusteredVariantAccessioningServiceTest.java @@ -15,28 +15,23 @@ */ package uk.ac.ebi.eva.accession.core.service.nonhuman; -import com.lordofthejars.nosqlunit.annotation.UsingDataSet; -import com.lordofthejars.nosqlunit.core.LoadStrategyEnum; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; -import org.junit.After; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.autoconfigure.orm.jpa.DataJpaTest; -import org.springframework.context.ApplicationContext; +import org.springframework.core.io.ResourceLoader; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.data.mongodb.core.query.Query; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.ampt2d.commons.accession.core.exceptions.AccessionDeprecatedException; import uk.ac.ebi.ampt2d.commons.accession.core.exceptions.AccessionDoesNotExistException; import uk.ac.ebi.ampt2d.commons.accession.core.exceptions.AccessionMergedException; import uk.ac.ebi.ampt2d.commons.accession.core.models.GetOrCreateAccessionWrapper; import uk.ac.ebi.ampt2d.commons.accession.hashing.SHA1HashingFunction; - import uk.ac.ebi.eva.accession.core.configuration.nonhuman.ClusteredVariantAccessioningConfiguration; import uk.ac.ebi.eva.accession.core.model.ClusteredVariant; import uk.ac.ebi.eva.accession.core.model.IClusteredVariant; @@ -44,7 +39,8 @@ import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpClusteredVariantOperationEntity; import uk.ac.ebi.eva.accession.core.summary.ClusteredVariantSummaryFunction; import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; -import uk.ac.ebi.eva.accession.core.test.rule.FixSpringMongoDbRule; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; +import uk.ac.ebi.eva.accession.core.utils.MongoTestDataLoader; import uk.ac.ebi.eva.commons.core.models.VariantType; import java.util.Arrays; @@ -52,26 +48,21 @@ import java.util.List; import java.util.function.Function; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; -@RunWith(SpringRunner.class) +@ExtendWith(SpringExtension.class) @DataJpaTest @TestPropertySource("classpath:rs-accession-test.properties") @ContextConfiguration(classes = {ClusteredVariantAccessioningConfiguration.class, MongoTestConfiguration.class}) -public class ClusteredVariantAccessioningServiceTest { +public class ClusteredVariantAccessioningServiceTest extends MongoTestContainerHelper { private static String TEST_APPLICATION_INSTANCE_ID = "test-application-instance-id"; - //Required by nosql-unit - @Autowired - private ApplicationContext applicationContext; - @Autowired private MongoTemplate mongoTemplate; - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName("clustered-variants-test").build()); + @Autowired + private ResourceLoader resourceLoader; @Autowired private ClusteredVariantAccessioningService service; @@ -79,13 +70,16 @@ public class ClusteredVariantAccessioningServiceTest { private Function clusteredHashingFunction = new ClusteredVariantSummaryFunction().andThen(new SHA1HashingFunction()); - @After + @BeforeEach + public void setUp() { + mongoTemplate.getDb().drop(); + } + + @AfterEach public void tearDown() throws Exception { - mongoTemplate.dropCollection(DbsnpClusteredVariantEntity.class); - mongoTemplate.dropCollection(DbsnpClusteredVariantOperationEntity.class); + mongoTemplate.getDb().drop(); } - @UsingDataSet(loadStrategy = LoadStrategyEnum.DELETE_ALL) @Test public void sameAccessionsAreReturnedForIdenticalVariants() throws Exception { List variants = Arrays.asList( @@ -121,9 +115,10 @@ public void mergeRemappedVariants() throws AccessionMergedException, AccessionDo assertEquals(1, mongoTemplate.count(new Query(), DbsnpClusteredVariantOperationEntity.class)); } - @UsingDataSet(locations = {"/test-data/dbsnpClusteredVariantEntity.json"}) @Test public void getAllByAccession() throws AccessionMergedException, AccessionDoesNotExistException, AccessionDeprecatedException { + new MongoTestDataLoader(mongoTemplate, resourceLoader).load("/test-data/dbsnpClusteredVariantEntity.json"); + assertEquals(2, service.getAllByAccession(1314L).size()); } } diff --git a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/service/nonhuman/SubmittedVariantAccessioningServiceTest.java b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/service/nonhuman/SubmittedVariantAccessioningServiceTest.java index 4c888303d..9a1b6c91a 100644 --- a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/service/nonhuman/SubmittedVariantAccessioningServiceTest.java +++ b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/service/nonhuman/SubmittedVariantAccessioningServiceTest.java @@ -15,24 +15,19 @@ */ package uk.ac.ebi.eva.accession.core.service.nonhuman; -import com.lordofthejars.nosqlunit.annotation.UsingDataSet; -import com.lordofthejars.nosqlunit.core.LoadStrategyEnum; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; import com.mongodb.client.MongoCursor; import org.bson.Document; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.autoconfigure.orm.jpa.DataJpaTest; -import org.springframework.context.ApplicationContext; -import org.springframework.data.mongodb.MongoDbFactory; +import org.springframework.core.io.ResourceLoader; +import org.springframework.data.mongodb.MongoDatabaseFactory; +import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.ampt2d.commons.accession.core.AccessionSaveMode; import uk.ac.ebi.ampt2d.commons.accession.core.exceptions.AccessionCouldNotBeGeneratedException; import uk.ac.ebi.ampt2d.commons.accession.core.exceptions.AccessionDeprecatedException; @@ -44,7 +39,6 @@ import uk.ac.ebi.ampt2d.commons.accession.core.models.GetOrCreateAccessionWrapper; import uk.ac.ebi.ampt2d.commons.accession.core.models.IEvent; import uk.ac.ebi.ampt2d.commons.accession.hashing.SHA1HashingFunction; - import uk.ac.ebi.eva.accession.core.configuration.nonhuman.SubmittedVariantAccessioningConfiguration; import uk.ac.ebi.eva.accession.core.generators.DbsnpMonotonicAccessionGenerator; import uk.ac.ebi.eva.accession.core.model.ISubmittedVariant; @@ -54,8 +48,10 @@ import uk.ac.ebi.eva.accession.core.service.nonhuman.dbsnp.DbsnpSubmittedVariantMonotonicAccessioningService; import uk.ac.ebi.eva.accession.core.service.nonhuman.eva.SubmittedVariantInactiveService; import uk.ac.ebi.eva.accession.core.summary.SubmittedVariantSummaryFunction; +import uk.ac.ebi.eva.accession.core.test.configuration.JPATestConfiguration; import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; -import uk.ac.ebi.eva.accession.core.test.rule.FixSpringMongoDbRule; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; +import uk.ac.ebi.eva.accession.core.utils.MongoTestDataLoader; import uk.ac.ebi.eva.commons.core.models.contigalias.ContigNamingConvention; import java.time.LocalDateTime; @@ -68,19 +64,20 @@ import java.util.NoSuchElementException; import java.util.stream.Collectors; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; import static uk.ac.ebi.eva.accession.core.model.ISubmittedVariant.DEFAULT_ALLELES_MATCH; import static uk.ac.ebi.eva.accession.core.model.ISubmittedVariant.DEFAULT_ASSEMBLY_MATCH; import static uk.ac.ebi.eva.accession.core.model.ISubmittedVariant.DEFAULT_SUPPORTED_BY_EVIDENCE; import static uk.ac.ebi.eva.accession.core.model.ISubmittedVariant.DEFAULT_VALIDATED; -@RunWith(SpringRunner.class) -@DataJpaTest +@ExtendWith(SpringExtension.class) @TestPropertySource("classpath:ss-accession-test.properties") -@ContextConfiguration(classes = {SubmittedVariantAccessioningConfiguration.class, MongoTestConfiguration.class}) -public class SubmittedVariantAccessioningServiceTest { +@ContextConfiguration(classes = {SubmittedVariantAccessioningConfiguration.class, MongoTestConfiguration.class, + JPATestConfiguration.class}) +public class SubmittedVariantAccessioningServiceTest extends MongoTestContainerHelper { private static String TEST_APPLICATION_INSTANCE_ID = "test-application-instance-id"; private static final Long CLUSTERED_VARIANT = null; @@ -121,10 +118,6 @@ public class SubmittedVariantAccessioningServiceTest { @Autowired private DbsnpSubmittedVariantAccessioningDatabaseService dbServiceDbsnp; - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName("submitted-variants-test").build()); - @Autowired private SubmittedVariantAccessioningService service; @@ -137,47 +130,49 @@ public class SubmittedVariantAccessioningServiceTest { @Autowired private Long accessioningMonotonicInitSs; - //Required by nosql-unit @Autowired - private ApplicationContext applicationContext; - - @Rule - public ExpectedException expectedException = ExpectedException.none(); + private MongoDatabaseFactory mongoDbFactory; @Autowired - private MongoDbFactory mongoDbFactory; + private ResourceLoader resourceLoader; + @Autowired + private MongoTemplate mongoTemplate; - @Before + @BeforeEach public void setUp() { + mongoTemplate.getDb().drop(); submittedVariant = new SubmittedVariant("GCA_000003055.3", 9913, PROJECT, "21", 20800319, "C", "T", - CLUSTERED_VARIANT, DEFAULT_SUPPORTED_BY_EVIDENCE, - DEFAULT_ASSEMBLY_MATCH, true, - DEFAULT_VALIDATED, null); + CLUSTERED_VARIANT, DEFAULT_SUPPORTED_BY_EVIDENCE, + DEFAULT_ASSEMBLY_MATCH, true, + DEFAULT_VALIDATED, null); newSubmittedVariant = new SubmittedVariant("assembly", 1111, "project", "contig_2", 100, "ref", "alt", - CLUSTERED_VARIANT, DEFAULT_SUPPORTED_BY_EVIDENCE, - DEFAULT_ASSEMBLY_MATCH, true, - DEFAULT_VALIDATED, null); + CLUSTERED_VARIANT, DEFAULT_SUPPORTED_BY_EVIDENCE, + DEFAULT_ASSEMBLY_MATCH, true, + DEFAULT_VALIDATED, null); dbsnpSubmittedVariant = new SubmittedVariant("GCA_000009999.3", 9999, PROJECT_DBSNP, "21", 20849999, "", "GG", - CLUSTERED_VARIANT, DEFAULT_SUPPORTED_BY_EVIDENCE, - DEFAULT_ASSEMBLY_MATCH, true, - DEFAULT_VALIDATED, null); + CLUSTERED_VARIANT, DEFAULT_SUPPORTED_BY_EVIDENCE, + DEFAULT_ASSEMBLY_MATCH, true, + DEFAULT_VALIDATED, null); submittedVariantModified = new SubmittedVariant("GCA_000003055.3", 9913, PROJECT, "21", 20800319, - "C", "TCTC", CLUSTERED_VARIANT, DEFAULT_SUPPORTED_BY_EVIDENCE, - DEFAULT_ASSEMBLY_MATCH, true, DEFAULT_VALIDATED, null); -// submittedVariantModified.setCreatedDate(LocalDateTime.now()); + "C", "TCTC", CLUSTERED_VARIANT, DEFAULT_SUPPORTED_BY_EVIDENCE, + DEFAULT_ASSEMBLY_MATCH, true, DEFAULT_VALIDATED, null); + } + + @AfterEach + public void tearDown() { + mongoTemplate.getDb().drop(); } - @UsingDataSet(loadStrategy = LoadStrategyEnum.DELETE_ALL) @Test public void sameAccessionsAreReturnedForIdenticalVariants() throws AccessionCouldNotBeGeneratedException { List variants = Arrays.asList( new SubmittedVariant("assembly", 1111, "project", "contig_1", 100, "ref", "alt", CLUSTERED_VARIANT, - DEFAULT_SUPPORTED_BY_EVIDENCE, DEFAULT_ASSEMBLY_MATCH, ALLELES_MATCH, - DEFAULT_VALIDATED, null), + DEFAULT_SUPPORTED_BY_EVIDENCE, DEFAULT_ASSEMBLY_MATCH, ALLELES_MATCH, + DEFAULT_VALIDATED, null), newSubmittedVariant); List> generatedAccessions = service.getOrCreate(variants, TEST_APPLICATION_INSTANCE_ID); @@ -186,13 +181,12 @@ public void sameAccessionsAreReturnedForIdenticalVariants() throws AccessionCoul assertEquals(new HashSet<>(generatedAccessions), new HashSet<>(retrievedAccessions)); } - @UsingDataSet(loadStrategy = LoadStrategyEnum.DELETE_ALL) @Test public void sameAccessionsAreReturnedForEquivalentVariants() throws AccessionCouldNotBeGeneratedException { List originalVariants = Arrays.asList( new SubmittedVariant("assembly", 1111, "project", "contig_1", 100, "ref", "alt", 10L, - DEFAULT_SUPPORTED_BY_EVIDENCE, DEFAULT_ASSEMBLY_MATCH, ALLELES_MATCH, - DEFAULT_VALIDATED, null), + DEFAULT_SUPPORTED_BY_EVIDENCE, DEFAULT_ASSEMBLY_MATCH, ALLELES_MATCH, + DEFAULT_VALIDATED, null), newSubmittedVariant); List requestedVariants = Arrays.asList( new SubmittedVariant("assembly", 1111, "project", "contig_1", 100, "ref", "alt", null), @@ -205,75 +199,86 @@ public void sameAccessionsAreReturnedForEquivalentVariants() throws AccessionCou assertEquals(new HashSet<>(generatedAccessions), new HashSet<>(retrievedAccessions)); } - @UsingDataSet(locations = {"/test-data/submittedVariantEntity.json", "/test-data/dbsnpSubmittedVariantEntity.json"}) @Test public void getOrCreateAccessionsInBothRepositories() throws AccessionCouldNotBeGeneratedException { + MongoTestDataLoader mongoTestDataLoader = new MongoTestDataLoader(mongoTemplate, resourceLoader); + mongoTestDataLoader.load("/test-data/submittedVariantEntity.json"); + mongoTestDataLoader.load("/test-data/dbsnpSubmittedVariantEntity.json"); + List variants = Arrays.asList(submittedVariant, newSubmittedVariant, dbsnpSubmittedVariant); List> submittedVariants = service.getOrCreate(variants, TEST_APPLICATION_INSTANCE_ID); long accession = submittedVariants.stream() - .filter(x -> x.getData().getProjectAccession().equals(PROJECT)) - .findAny().orElseThrow(NoSuchElementException::new) - .getAccession(); + .filter(x -> x.getData().getProjectAccession().equals(PROJECT)) + .findAny().orElseThrow(NoSuchElementException::new) + .getAccession(); assertEquals(ACCESSION, accession); long accessionDbsnp = submittedVariants.stream() - .filter(x -> x.getData().getProjectAccession().equals(PROJECT_DBSNP)) - .findAny().orElseThrow(NoSuchElementException::new) - .getAccession(); + .filter(x -> x.getData().getProjectAccession().equals(PROJECT_DBSNP)) + .findAny().orElseThrow(NoSuchElementException::new) + .getAccession(); assertEquals(ACCESSION_DBSNP_1, accessionDbsnp); assertEquals(3, submittedVariants.size()); } - @UsingDataSet(locations = {"/test-data/dbsnpSubmittedVariantEntity.json"}) @Test public void getOrCreateDbsnpIdenticalVariants() throws AccessionCouldNotBeGeneratedException { + new MongoTestDataLoader(mongoTemplate, resourceLoader).load("/test-data/dbsnpSubmittedVariantEntity.json"); + SubmittedVariant variantPresentInDbsnp = new SubmittedVariant("GCA_000009999.3", 9999, PROJECT_DBSNP, "21", - 20849999, "", "GG", null); + 20849999, "", "GG", null); List> generatedAccessions = service.getOrCreate( Collections.singletonList(variantPresentInDbsnp), TEST_APPLICATION_INSTANCE_ID); assertEquals(Collections.singleton(ACCESSION_DBSNP_1), - generatedAccessions.stream().map(AccessionWrapper::getAccession).collect(Collectors.toSet())); + generatedAccessions.stream().map(AccessionWrapper::getAccession).collect(Collectors.toSet())); } - @UsingDataSet(locations = {"/test-data/dbsnpSubmittedVariantEntity.json"}) @Test public void getOrCreateDbsnpEquivalentVariants() throws AccessionCouldNotBeGeneratedException { + new MongoTestDataLoader(mongoTemplate, resourceLoader).load("/test-data/dbsnpSubmittedVariantEntity.json"); + SubmittedVariant identicalVariantPresentInDbsnp = new SubmittedVariant("GCA_000009999.3", 9999, PROJECT_DBSNP, - "21", 20849999, "", "GG", null); + "21", 20849999, "", "GG", null); List> generatedAccessions = service.getOrCreate( Arrays.asList(identicalVariantPresentInDbsnp), TEST_APPLICATION_INSTANCE_ID); assertEquals(Collections.singleton(ACCESSION_DBSNP_1), - generatedAccessions.stream().map(AccessionWrapper::getAccession).collect(Collectors.toSet())); + generatedAccessions.stream().map(AccessionWrapper::getAccession).collect(Collectors.toSet())); int differentTaxonomy = 1000; SubmittedVariant equivalentVariantPresentInDbsnp = new SubmittedVariant("GCA_000009999.3", differentTaxonomy, - PROJECT_DBSNP, "21", 20849999, "", "GG", - null); + PROJECT_DBSNP, "21", 20849999, "", "GG", + null); List> generatedAccessions2 = service.getOrCreate( Arrays.asList(identicalVariantPresentInDbsnp, equivalentVariantPresentInDbsnp), TEST_APPLICATION_INSTANCE_ID); assertEquals(Collections.singleton(ACCESSION_DBSNP_1), - generatedAccessions2.stream().map(AccessionWrapper::getAccession).collect(Collectors.toSet())); + generatedAccessions2.stream().map(AccessionWrapper::getAccession).collect(Collectors.toSet())); } - @UsingDataSet(locations = {"/test-data/submittedVariantEntity.json", "/test-data/dbsnpSubmittedVariantEntity.json"}) @Test public void getFromBothRepositories() { + MongoTestDataLoader mongoTestDataLoader = new MongoTestDataLoader(mongoTemplate, resourceLoader); + mongoTestDataLoader.load("/test-data/submittedVariantEntity.json"); + mongoTestDataLoader.load("/test-data/dbsnpSubmittedVariantEntity.json"); + List variants = Arrays.asList(submittedVariant, newSubmittedVariant, dbsnpSubmittedVariant); List> accessions = service.get(variants); assertEquals(2, accessions.size()); } - @UsingDataSet(locations = {"/test-data/submittedVariantEntity.json", "/test-data/dbsnpSubmittedVariantEntity.json"}) @Test public void getByAccessionsFromBothRepositories() throws AccessionMergedException, AccessionDoesNotExistException, AccessionDeprecatedException { + MongoTestDataLoader mongoTestDataLoader = new MongoTestDataLoader(mongoTemplate, resourceLoader); + mongoTestDataLoader.load("/test-data/submittedVariantEntity.json"); + mongoTestDataLoader.load("/test-data/dbsnpSubmittedVariantEntity.json"); + AccessionWrapper retrievedSubmittedVariant = service.getByAccession(ACCESSION); assertEquals(submittedVariant, retrievedSubmittedVariant.getData()); AccessionWrapper retrievedSubmittedDbsnpVariant = service.getByAccession( @@ -281,23 +286,28 @@ public void getByAccessionsFromBothRepositories() assertEquals(dbsnpSubmittedVariant, retrievedSubmittedDbsnpVariant.getData()); } - @UsingDataSet(locations = {"/test-data/dbsnpSubmittedVariantEntity.json"}) @Test public void getAllByAccession() throws AccessionMergedException, AccessionDoesNotExistException, AccessionDeprecatedException { + new MongoTestDataLoader(mongoTemplate, resourceLoader).load("/test-data/dbsnpSubmittedVariantEntity.json"); + assertEquals(2, service.getAllByAccession(2200000002L).size()); } - @UsingDataSet(locations = {"/test-data/dbsnpSubmittedVariantEntity.json"}) @Test public void getAllByIdFields() { + new MongoTestDataLoader(mongoTemplate, resourceLoader).load("/test-data/dbsnpSubmittedVariantEntity.json"); + assertEquals(2, service.getAllByIdFields("GCA_000009999.3", "21", Arrays.asList("DBSNP999", "DBSNP111"), - 20849999L, "", "GG", ContigNamingConvention.INSDC).size()); + 20849999L, "", "GG", ContigNamingConvention.INSDC).size()); } - @UsingDataSet(locations = {"/test-data/submittedVariantEntity.json", "/test-data/dbsnpSubmittedVariantEntity.json"}) @Test public void getByAccessionAndVersionFromBothRepositories() throws AccessionMergedException, AccessionDoesNotExistException, AccessionDeprecatedException { + MongoTestDataLoader mongoTestDataLoader = new MongoTestDataLoader(mongoTemplate, resourceLoader); + mongoTestDataLoader.load("/test-data/submittedVariantEntity.json"); + mongoTestDataLoader.load("/test-data/dbsnpSubmittedVariantEntity.json"); + AccessionWrapper retrievedSubmittedVariant = service.getByAccessionAndVersion( ACCESSION, 1); assertEquals(submittedVariant, retrievedSubmittedVariant.getData()); @@ -307,10 +317,11 @@ public void getByAccessionAndVersionFromBothRepositories() throws AccessionMerge assertEquals(dbsnpSubmittedVariant, retrievedDbsnpSubmittedVariant.getData()); } - @UsingDataSet(locations = {"/test-data/submittedVariantEntity.json"}) @Test public void updateSubmittedVariant() throws AccessionDeprecatedException, AccessionDoesNotExistException, AccessionMergedException, HashAlreadyExistsException { + new MongoTestDataLoader(mongoTemplate, resourceLoader).load("/test-data/submittedVariantEntity.json"); + service.update(ACCESSION, 1, submittedVariantModified); assertVariantUpdated(ACCESSION, submittedVariantModified); } @@ -330,18 +341,20 @@ private void assertVariantUpdated(long accession, ISubmittedVariant modifiedVari assertNull(lastOperation.getMergedInto()); } - @UsingDataSet(locations = {"/test-data/dbsnpSubmittedVariantEntity.json"}) @Test public void updateDbsnpSubmittedVariant() throws AccessionDeprecatedException, AccessionDoesNotExistException, AccessionMergedException, HashAlreadyExistsException { + new MongoTestDataLoader(mongoTemplate, resourceLoader).load("/test-data/dbsnpSubmittedVariantEntity.json"); + service.update(ACCESSION_DBSNP_1, 1, submittedVariantModified); assertVariantUpdated(ACCESSION_DBSNP_1, submittedVariantModified); } - @UsingDataSet(locations = {"/test-data/submittedVariantEntity.json"}) @Test public void patchSubmittedVariant() throws AccessionDeprecatedException, AccessionDoesNotExistException, AccessionMergedException, HashAlreadyExistsException { + new MongoTestDataLoader(mongoTemplate, resourceLoader).load("/test-data/submittedVariantEntity.json"); + service.patch(ACCESSION, submittedVariantModified); assertVariantPatched(ACCESSION, submittedVariantModified); } @@ -352,25 +365,26 @@ private void assertVariantPatched(long accession, ISubmittedVariant modifiedVari assertEquals(modifiedVariant, variantFromDb); } - @UsingDataSet(locations = {"/test-data/dbsnpSubmittedVariantEntity.json"}) @Test public void patchDbsnpSubmittedVariant() throws AccessionDeprecatedException, AccessionDoesNotExistException, AccessionMergedException, HashAlreadyExistsException { + new MongoTestDataLoader(mongoTemplate, resourceLoader).load("/test-data/dbsnpSubmittedVariantEntity.json"); + service.patch(ACCESSION_DBSNP_1, submittedVariantModified); assertVariantPatched(ACCESSION_DBSNP_1, submittedVariantModified); } - @UsingDataSet(locations = {"/test-data/submittedVariantEntity.json"}) @Test public void deprecateSubmittedVariant() throws AccessionMergedException, AccessionDoesNotExistException, AccessionDeprecatedException { + new MongoTestDataLoader(mongoTemplate, resourceLoader).load("/test-data/submittedVariantEntity.json"); + assertNotNull(service.getByAccession(ACCESSION_TO_DEPRECATE)); service.deprecate(ACCESSION_TO_DEPRECATE, DEPRECATE_REASON); assertVariantDeprecated(ACCESSION_TO_DEPRECATE, DEPRECATE_REASON); } - private void assertVariantDeprecated(long accession, String reason) - throws AccessionMergedException, AccessionDoesNotExistException, AccessionDeprecatedException { + private void assertVariantDeprecated(long accession, String reason) { IEvent lastOperation; if (accession >= accessioningMonotonicInitSs) { lastOperation = inactiveService.getLastEvent(accession); @@ -382,23 +396,24 @@ private void assertVariantDeprecated(long accession, String reason) assertNull(lastOperation.getMergedInto()); assertEquals(reason, lastOperation.getReason()); - expectedException.expect(AccessionDeprecatedException.class); - service.getByAccession(accession); + assertThrows(AccessionDeprecatedException.class, () -> service.getByAccession(accession)); } - @UsingDataSet(locations = {"/test-data/dbsnpSubmittedVariantEntity.json"}) @Test public void deprecateDnsnpSubmittedVariant() throws AccessionMergedException, AccessionDoesNotExistException, AccessionDeprecatedException { + new MongoTestDataLoader(mongoTemplate, resourceLoader).load("/test-data/dbsnpSubmittedVariantEntity.json"); + assertNotNull(service.getByAccession(ACCESSION_DBSNP_1)); service.deprecate(ACCESSION_DBSNP_1, DEPRECATE_REASON); assertVariantDeprecated(ACCESSION_DBSNP_1, DEPRECATE_REASON); } - @UsingDataSet(locations = {"/test-data/submittedVariantEntity.json"}) @Test public void mergeSubmittedVariant() throws AccessionMergedException, AccessionDoesNotExistException, AccessionDeprecatedException { + new MongoTestDataLoader(mongoTemplate, resourceLoader).load("/test-data/submittedVariantEntity.json"); + assertNotNull(service.getByAccession(ACCESSION_TO_MERGE_1)); assertNotNull(service.getByAccession(ACCESSION_TO_MERGE_2)); service.merge(ACCESSION_TO_MERGE_1, ACCESSION_TO_MERGE_2, MERGE_REASON); @@ -420,49 +435,47 @@ private void assertVariantMerged(long accessionOrigin, long accessionDestination assertNotNull(service.getByAccession(accessionDestination)); - expectedException.expect(AccessionMergedException.class); - service.getByAccession(accessionOrigin); + assertThrows(AccessionMergedException.class, () -> service.getByAccession(accessionOrigin)); } - @UsingDataSet(locations = {"/test-data/dbsnpSubmittedVariantEntity.json"}) @Test public void mergeDbsnpSubmittedVariant() throws AccessionMergedException, AccessionDoesNotExistException, AccessionDeprecatedException { + new MongoTestDataLoader(mongoTemplate, resourceLoader).load("/test-data/dbsnpSubmittedVariantEntity.json"); + assertNotNull(service.getByAccession(ACCESSION_DBSNP_1)); assertNotNull(service.getByAccession(ACCESSION_DBSNP_2)); service.merge(ACCESSION_DBSNP_1, ACCESSION_DBSNP_2, MERGE_REASON); assertVariantMerged(ACCESSION_DBSNP_1, ACCESSION_DBSNP_2, MERGE_REASON); } - @UsingDataSet(loadStrategy = LoadStrategyEnum.DELETE_ALL) - @Test(expected = UnsupportedOperationException.class) - public void exceptionWhenCreateAccessionForDbsnpVariant() throws AccessionCouldNotBeGeneratedException { + @Test + public void exceptionWhenCreateAccessionForDbsnpVariant() { DbsnpSubmittedVariantMonotonicAccessioningService accessioningServiceDbsnp = new DbsnpSubmittedVariantMonotonicAccessioningService( dbsnpAccessionGenerator, dbServiceDbsnp, new SubmittedVariantSummaryFunction(), new SHA1HashingFunction(), AccessionSaveMode.SAVE_ALL_THEN_RESOLVE); - accessioningServiceDbsnp.getOrCreate(Collections.singletonList(submittedVariant), TEST_APPLICATION_INSTANCE_ID); + assertThrows(UnsupportedOperationException.class, () -> accessioningServiceDbsnp.getOrCreate(Collections.singletonList(submittedVariant), TEST_APPLICATION_INSTANCE_ID)); } - @UsingDataSet(loadStrategy = LoadStrategyEnum.DELETE_ALL) @Test public void defaultValuesAreNotStored() throws AccessionCouldNotBeGeneratedException { boolean NOT_DEFAULT_SUPPORTED_BY_EVIDENCE = !DEFAULT_SUPPORTED_BY_EVIDENCE; SubmittedVariant submittedVariant = new SubmittedVariant("assembly", 1111, "project", "contig_1", 100, "ref", - "alt", CLUSTERED_VARIANT, - NOT_DEFAULT_SUPPORTED_BY_EVIDENCE, - DEFAULT_ASSEMBLY_MATCH, - DEFAULT_ALLELES_MATCH, - DEFAULT_VALIDATED, null); + "alt", CLUSTERED_VARIANT, + NOT_DEFAULT_SUPPORTED_BY_EVIDENCE, + DEFAULT_ASSEMBLY_MATCH, + DEFAULT_ALLELES_MATCH, + DEFAULT_VALIDATED, null); service.getOrCreate(Collections.singletonList(submittedVariant), TEST_APPLICATION_INSTANCE_ID); - List submittedVariantEntities = new ArrayList<>(); - MongoCursor submittedVariantEntitiesCursor = mongoDbFactory.getDb() - .getCollection("submittedVariantEntity") - .find().iterator(); + List submittedVariantEntities = new ArrayList<>(); + MongoCursor submittedVariantEntitiesCursor = mongoDbFactory.getMongoDatabase() + .getCollection("submittedVariantEntity") + .find().iterator(); submittedVariantEntitiesCursor.forEachRemaining(submittedVariantEntities::add); assertEquals(NOT_DEFAULT_SUPPORTED_BY_EVIDENCE, submittedVariantEntities.get(0).get("evidence")); @@ -471,20 +484,21 @@ public void defaultValuesAreNotStored() throws AccessionCouldNotBeGeneratedExcep assertEquals(null, submittedVariantEntities.get(0).get("validated")); } - @UsingDataSet(locations = {"/test-data/dbsnpSubmittedVariantEntity.json"}) @Test public void getSeveralVariantsWithSameAccession() { + new MongoTestDataLoader(mongoTemplate, resourceLoader).load("/test-data/dbsnpSubmittedVariantEntity.json"); + SubmittedVariant multiallelicSubmittedVariant1 = new SubmittedVariant("GCA_000009999.3", 9999, "DBSNP555", - "21", 20849999, "A", "G", 2200000002L, - DEFAULT_SUPPORTED_BY_EVIDENCE, - DEFAULT_ASSEMBLY_MATCH, - true, DEFAULT_VALIDATED, null); + "21", 20849999, "A", "G", 2200000002L, + DEFAULT_SUPPORTED_BY_EVIDENCE, + DEFAULT_ASSEMBLY_MATCH, + true, DEFAULT_VALIDATED, null); SubmittedVariant multiallelicSubmittedVariant2 = new SubmittedVariant("GCA_000009999.3", 9999, "DBSNP555", - "21", 20849999, "A", "C", 2200000003L, - DEFAULT_SUPPORTED_BY_EVIDENCE, - DEFAULT_ASSEMBLY_MATCH, - true, DEFAULT_VALIDATED, null); + "21", 20849999, "A", "C", 2200000003L, + DEFAULT_SUPPORTED_BY_EVIDENCE, + DEFAULT_ASSEMBLY_MATCH, + true, DEFAULT_VALIDATED, null); List> accessions = service.get( Arrays.asList(multiallelicSubmittedVariant1, multiallelicSubmittedVariant2)); @@ -492,7 +506,6 @@ public void getSeveralVariantsWithSameAccession() { assertEquals(1, accessions.stream().map(AccessionWrapper::getAccession).collect(Collectors.toSet()).size()); } - @UsingDataSet(loadStrategy = LoadStrategyEnum.DELETE_ALL) @Test public void createdDateIsReturnedInAccessionedVariant() throws AccessionCouldNotBeGeneratedException { LocalDateTime createdDate = LocalDateTime.of(2018, Month.SEPTEMBER, 18, 9, 0); diff --git a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/service/nonhuman/eva/ClusteredVariantMonotonicAccessioningServiceTest.java b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/service/nonhuman/eva/ClusteredVariantMonotonicAccessioningServiceTest.java index 3686a4df2..e94ad0347 100644 --- a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/service/nonhuman/eva/ClusteredVariantMonotonicAccessioningServiceTest.java +++ b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/service/nonhuman/eva/ClusteredVariantMonotonicAccessioningServiceTest.java @@ -17,42 +17,61 @@ */ package uk.ac.ebi.eva.accession.core.service.nonhuman.eva; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.autoconfigure.orm.jpa.DataJpaTest; +import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.ampt2d.commons.accession.core.exceptions.AccessionCouldNotBeGeneratedException; import uk.ac.ebi.ampt2d.commons.accession.core.models.GetOrCreateAccessionWrapper; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.ClusteredVariantAccessioningConfiguration; import uk.ac.ebi.eva.accession.core.model.ClusteredVariant; import uk.ac.ebi.eva.accession.core.model.IClusteredVariant; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.commons.core.models.VariantType; import java.util.Arrays; import java.util.List; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; -@RunWith(SpringRunner.class) +@ExtendWith(SpringExtension.class) @DataJpaTest @TestPropertySource("classpath:rs-accession-test.properties") -@ContextConfiguration(classes = {ClusteredVariantAccessioningConfiguration.class}) -public class ClusteredVariantMonotonicAccessioningServiceTest { +@ContextConfiguration(classes = {ClusteredVariantAccessioningConfiguration.class, MongoTestConfiguration.class}) +public class ClusteredVariantMonotonicAccessioningServiceTest extends MongoTestContainerHelper { @Autowired private ClusteredVariantMonotonicAccessioningService clusteredVariantMonotonicAccessioningService; + @Autowired + private MongoTemplate mongoTemplate; + + @BeforeEach + public void setUp() throws Exception { + mongoTemplate.getDb().drop(); + } + + @AfterEach + void cleanDb() { + mongoTemplate.getDb().drop(); + } + + @Test public void service() throws AccessionCouldNotBeGeneratedException { ClusteredVariant clusteredVariant1 = new ClusteredVariant("asm1", 100, "chr1", 1, VariantType.SNV, false, null); ClusteredVariant clusteredVariant2 = new ClusteredVariant("asm1", 100, "chr1", 5, VariantType.SNV, false, null); List> accessionedWrappers = clusteredVariantMonotonicAccessioningService.getOrCreate(Arrays.asList(clusteredVariant1, - clusteredVariant2), "test-application-instance-id"); + clusteredVariant2), "test-application-instance-id"); assertEquals(2, accessionedWrappers.size()); assertTrue(isAccessionInResults(accessionedWrappers, 3000000000L)); assertTrue(isAccessionInResults(accessionedWrappers, 3000000001L)); diff --git a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/summary/ClusteredVariantSummaryFunctionTest.java b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/summary/ClusteredVariantSummaryFunctionTest.java index 35ef54bc9..99eb79f38 100644 --- a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/summary/ClusteredVariantSummaryFunctionTest.java +++ b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/summary/ClusteredVariantSummaryFunctionTest.java @@ -17,8 +17,8 @@ */ package uk.ac.ebi.eva.accession.core.summary; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import uk.ac.ebi.eva.accession.core.model.ClusteredVariant; import uk.ac.ebi.eva.accession.core.model.IClusteredVariant; @@ -27,8 +27,8 @@ import java.time.LocalDateTime; import java.time.Month; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; public class ClusteredVariantSummaryFunctionTest { @@ -46,7 +46,7 @@ public class ClusteredVariantSummaryFunctionTest { private ClusteredVariantSummaryFunction summaryFunction; - @Before + @BeforeEach public void setUp() { summaryFunction = new ClusteredVariantSummaryFunction(); } diff --git a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/summary/SubmittedVariantSummaryFunctionTest.java b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/summary/SubmittedVariantSummaryFunctionTest.java index defe2cef1..238b669dd 100644 --- a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/summary/SubmittedVariantSummaryFunctionTest.java +++ b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/summary/SubmittedVariantSummaryFunctionTest.java @@ -17,8 +17,8 @@ */ package uk.ac.ebi.eva.accession.core.summary; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import uk.ac.ebi.eva.accession.core.model.ISubmittedVariant; import uk.ac.ebi.eva.accession.core.model.SubmittedVariant; @@ -26,8 +26,8 @@ import java.time.LocalDateTime; import java.time.Month; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; public class SubmittedVariantSummaryFunctionTest { @@ -57,7 +57,7 @@ public class SubmittedVariantSummaryFunctionTest { private SubmittedVariantSummaryFunction summaryFunction; - @Before + @BeforeEach public void setUp() { summaryFunction = new SubmittedVariantSummaryFunction(); } diff --git a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/test/configuration/DbsnpTestDataSource.java b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/test/configuration/DbsnpTestDataSource.java index dda176d9e..392f6c409 100644 --- a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/test/configuration/DbsnpTestDataSource.java +++ b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/test/configuration/DbsnpTestDataSource.java @@ -23,7 +23,6 @@ import org.springframework.jdbc.datasource.init.DatabasePopulator; import org.springframework.jdbc.datasource.init.DatabasePopulatorUtils; import org.springframework.jdbc.datasource.init.ResourceDatabasePopulator; - import uk.ac.ebi.eva.accession.core.configuration.DbsnpDataSource; import javax.sql.DataSource; diff --git a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/test/configuration/JPATestConfiguration.java b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/test/configuration/JPATestConfiguration.java new file mode 100644 index 000000000..66a846d06 --- /dev/null +++ b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/test/configuration/JPATestConfiguration.java @@ -0,0 +1,41 @@ +package uk.ac.ebi.eva.accession.core.test.configuration; + +import jakarta.persistence.EntityManagerFactory; +import org.springframework.context.annotation.Bean; +import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseBuilder; +import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseType; +import org.springframework.orm.jpa.JpaTransactionManager; +import org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean; +import org.springframework.orm.jpa.vendor.HibernateJpaVendorAdapter; +import org.springframework.transaction.PlatformTransactionManager; + +import javax.sql.DataSource; + +public class JPATestConfiguration { + @Bean + public DataSource dataSource() { + return new EmbeddedDatabaseBuilder() + .setType(EmbeddedDatabaseType.HSQL) + .build(); + } + + @Bean + public LocalContainerEntityManagerFactoryBean entityManagerFactory(DataSource dataSource) { + LocalContainerEntityManagerFactoryBean em = new LocalContainerEntityManagerFactoryBean(); + em.setDataSource(dataSource); + // Package where ContiguousIdBlockEntity (the JPA entity) lives + em.setPackagesToScan( + "uk.ac.ebi.ampt2d.commons.accession.persistence.jpa.monotonic.entities" + ); + HibernateJpaVendorAdapter vendorAdapter = new HibernateJpaVendorAdapter(); + vendorAdapter.setGenerateDdl(true); // auto-creates the schema in H2 + vendorAdapter.setShowSql(false); + em.setJpaVendorAdapter(vendorAdapter); + return em; + } + + @Bean + public PlatformTransactionManager transactionManager(EntityManagerFactory entityManagerFactory) { + return new JpaTransactionManager(entityManagerFactory); + } +} diff --git a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/test/configuration/human/MongoHumanTestConfiguration.java b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/test/configuration/human/MongoHumanTestConfiguration.java index 773f7eb12..ac6ddcaa7 100644 --- a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/test/configuration/human/MongoHumanTestConfiguration.java +++ b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/test/configuration/human/MongoHumanTestConfiguration.java @@ -17,22 +17,89 @@ */ package uk.ac.ebi.eva.accession.core.test.configuration.human; +import com.mongodb.ReadPreference; +import com.mongodb.WriteConcern; +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoClients; +import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.domain.EntityScan; import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Import; +import org.springframework.context.annotation.Primary; +import org.springframework.data.mongodb.MongoDatabaseFactory; +import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.data.mongodb.core.SimpleMongoClientDatabaseFactory; +import org.springframework.data.mongodb.core.WriteResultChecking; +import org.springframework.data.mongodb.core.convert.DefaultDbRefResolver; +import org.springframework.data.mongodb.core.convert.DefaultMongoTypeMapper; +import org.springframework.data.mongodb.core.convert.MappingMongoConverter; +import org.springframework.data.mongodb.core.convert.MongoCustomConversions; +import org.springframework.data.mongodb.core.mapping.MongoMappingContext; import org.springframework.data.mongodb.repository.config.EnableMongoRepositories; import uk.ac.ebi.eva.accession.core.configuration.ApplicationPropertiesConfiguration; import uk.ac.ebi.eva.accession.core.configuration.human.HumanClusteredVariantAccessioningConfiguration; import uk.ac.ebi.eva.accession.core.configuration.human.HumanMongoConfiguration; +import java.util.Collections; + @Configuration @EntityScan(basePackages = {"uk.ac.ebi.eva.accession.core.repository.human"}) @EnableMongoRepositories(basePackages = "uk.ac.ebi.eva.accession.core.repository.human", mongoTemplateRef = "humanMongoTemplate") @Import({HumanMongoConfiguration.class, HumanClusteredVariantAccessioningConfiguration.class, ApplicationPropertiesConfiguration.class - }) +}) @EnableConfigurationProperties(HumanMongoConfiguration.class) public class MongoHumanTestConfiguration { + + @Value("${spring.data.mongodb.uri}") + private String mongoUri; + + @Value("${spring.data.mongodb.database}") + private String database; + + @Value("${mongodb.read-preference:primary}") + private String readPreference; + + @Bean + @Primary + public MongoCustomConversions mongoCustomConversions() { + return new MongoCustomConversions(Collections.emptyList()); + } + + @Bean + @Primary + public MongoMappingContext mongoMappingContext(MongoCustomConversions conversions) { + MongoMappingContext context = new MongoMappingContext(); + context.setSimpleTypeHolder(conversions.getSimpleTypeHolder()); + context.setAutoIndexCreation(false); + return context; + } + + + @Bean("humanMongoClient") + public MongoClient humanMongoClient() { + return MongoClients.create(mongoUri); + } + + @Bean("humanFactory") + public MongoDatabaseFactory humanMongoDbFactory() { + return new SimpleMongoClientDatabaseFactory(humanMongoClient(), database); + } + + @Bean("humanMongoTemplate") + public MongoTemplate humanMongoTemplate() { + MongoMappingContext context = new MongoMappingContext(); + MappingMongoConverter converter = new MappingMongoConverter( + new DefaultDbRefResolver(humanMongoDbFactory()), context); + converter.setTypeMapper(new DefaultMongoTypeMapper(null)); + converter.afterPropertiesSet(); + MongoTemplate template = new MongoTemplate(humanMongoDbFactory(), converter); + template.setReadPreference(ReadPreference.valueOf(readPreference)); + template.setWriteConcern(WriteConcern.MAJORITY); + template.setWriteResultChecking(WriteResultChecking.EXCEPTION); + return template; + } } diff --git a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/test/configuration/nonhuman/MongoTestConfiguration.java b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/test/configuration/nonhuman/MongoTestConfiguration.java index 4d883e37f..1a357586c 100644 --- a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/test/configuration/nonhuman/MongoTestConfiguration.java +++ b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/test/configuration/nonhuman/MongoTestConfiguration.java @@ -17,12 +17,30 @@ */ package uk.ac.ebi.eva.accession.core.test.configuration.nonhuman; +import com.mongodb.ReadPreference; +import com.mongodb.WriteConcern; +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoClients; +import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.domain.EntityScan; import org.springframework.boot.test.autoconfigure.data.mongo.AutoConfigureDataMongo; +import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Primary; +import org.springframework.data.mongodb.MongoDatabaseFactory; import org.springframework.data.mongodb.config.EnableMongoAuditing; +import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.data.mongodb.core.SimpleMongoClientDatabaseFactory; +import org.springframework.data.mongodb.core.WriteResultChecking; +import org.springframework.data.mongodb.core.convert.DefaultDbRefResolver; +import org.springframework.data.mongodb.core.convert.DefaultMongoTypeMapper; +import org.springframework.data.mongodb.core.convert.MappingMongoConverter; +import org.springframework.data.mongodb.core.convert.MongoCustomConversions; +import org.springframework.data.mongodb.core.mapping.MongoMappingContext; import org.springframework.data.mongodb.repository.config.EnableMongoRepositories; +import java.util.Collections; + @Configuration @EntityScan(basePackages = {"uk.ac.ebi.eva.accession.core.persistence"}) @EnableMongoRepositories(basePackages = { @@ -32,4 +50,56 @@ @AutoConfigureDataMongo public class MongoTestConfiguration { + @Value("${spring.data.mongodb.uri}") + private String mongoUri; + + @Value("${spring.data.mongodb.database}") + private String database; + + @Value("${mongodb.read-preference:primary}") + private String readPreference; + + @Bean + @Primary + public MongoClient mongoClient() { + return MongoClients.create(mongoUri); + } + + @Bean("primaryFactory") + @Primary + public MongoDatabaseFactory mongoDbFactory() { + return new SimpleMongoClientDatabaseFactory(mongoClient(), database); + } + + @Bean + @Primary + public MongoCustomConversions mongoCustomConversions() { + return new MongoCustomConversions(Collections.emptyList()); + } + + @Bean + @Primary + public MongoMappingContext mongoMappingContext(MongoCustomConversions conversions) { + MongoMappingContext context = new MongoMappingContext(); + context.setSimpleTypeHolder(conversions.getSimpleTypeHolder()); + context.setAutoIndexCreation(false); + return context; + } + + @Bean + @Primary + public MongoTemplate mongoTemplate(MongoMappingContext mongoMappingContext, + MongoCustomConversions conversions) { + MappingMongoConverter converter = new MappingMongoConverter( + new DefaultDbRefResolver(mongoDbFactory()), mongoMappingContext); + converter.setCustomConversions(conversions); + converter.setTypeMapper(new DefaultMongoTypeMapper(null)); + converter.afterPropertiesSet(); + MongoTemplate mongoTemplate = new MongoTemplate(mongoDbFactory(), converter); + mongoTemplate.setReadPreference(ReadPreference.valueOf(readPreference)); + mongoTemplate.setWriteConcern(WriteConcern.MAJORITY); + mongoTemplate.setWriteResultChecking(WriteResultChecking.EXCEPTION); + return mongoTemplate; + } + } diff --git a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/test/rule/FixSpringMongoDbRule.java b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/test/rule/FixSpringMongoDbRule.java deleted file mode 100644 index c45eff3d4..000000000 --- a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/test/rule/FixSpringMongoDbRule.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * - * Copyright 2018 EMBL - European Bioinformatics Institute - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -package uk.ac.ebi.eva.accession.core.test.rule; - -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfiguration; -import com.lordofthejars.nosqlunit.mongodb.SpringMongoDbRule; - -/** - * Temporary fix until nosql unit rc-6 or final is released - */ -public class FixSpringMongoDbRule extends SpringMongoDbRule { - - public FixSpringMongoDbRule(MongoDbConfiguration mongoDbConfiguration) { - super(mongoDbConfiguration); - } - - public FixSpringMongoDbRule(MongoDbConfiguration mongoDbConfiguration, Object object) { - super(mongoDbConfiguration, object); - } - - @Override - public void close() { - // DO NOT CLOSE the connection (Spring will do it when destroying the context) - } - -} diff --git a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/utils/MongoTestContainerHelper.java b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/utils/MongoTestContainerHelper.java new file mode 100644 index 000000000..c3bc47b4f --- /dev/null +++ b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/utils/MongoTestContainerHelper.java @@ -0,0 +1,24 @@ +package uk.ac.ebi.eva.accession.core.utils; + +import org.springframework.boot.testcontainers.service.connection.ServiceConnection; +import org.springframework.test.context.DynamicPropertyRegistry; +import org.springframework.test.context.DynamicPropertySource; +import org.testcontainers.containers.MongoDBContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +@Testcontainers +public abstract class MongoTestContainerHelper { + + private static final String MONGO_IMAGE = "mongo:6.0"; + + @Container + @ServiceConnection + public static MongoDBContainer mongo = new MongoDBContainer(MONGO_IMAGE); + + @DynamicPropertySource + static void mongoProperties(DynamicPropertyRegistry registry) { + registry.add("spring.data.mongodb.uri", mongo::getConnectionString); + registry.add("human.mongodb.uri", mongo::getConnectionString); + } +} \ No newline at end of file diff --git a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/utils/MongoTestDataLoader.java b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/utils/MongoTestDataLoader.java new file mode 100644 index 000000000..444cda41b --- /dev/null +++ b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/utils/MongoTestDataLoader.java @@ -0,0 +1,106 @@ +package uk.ac.ebi.eva.accession.core.utils; + +import com.mongodb.client.MongoCollection; +import org.bson.BsonDocument; +import org.bson.BsonValue; +import org.springframework.core.io.Resource; +import org.springframework.core.io.ResourceLoader; +import org.springframework.data.mongodb.core.MongoTemplate; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class MongoTestDataLoader { + private final MongoTemplate mongoTemplate; + private final ResourceLoader resourceLoader; + + public MongoTestDataLoader(MongoTemplate mongoTemplate, ResourceLoader resourceLoader) { + this.mongoTemplate = mongoTemplate; + this.resourceLoader = resourceLoader; + } + + public void load(String resourcePath) { + load(resourcePath, null); + } + + public void load(String resourcePath, String collectionName) { + load(resourcePath, collectionName, false); + } + + public void load(String resourcePath, String collectionName, boolean dropCollection) { + try { + BsonDocument bsonRoot = readBsonRoot(resourcePath); + + String resolvedCollectionName; + BsonValue dataValue; + + if (collectionName != null && bsonRoot.containsKey(collectionName)) { + resolvedCollectionName = collectionName; + dataValue = bsonRoot.get(collectionName); + } else if (bsonRoot.size() == 1) { + Map.Entry entry = bsonRoot.entrySet().iterator().next(); + resolvedCollectionName = entry.getKey(); + dataValue = entry.getValue(); + } else { + throw new IllegalArgumentException("JSON file has multiple collections but no collectionName was specified. " + + "Use loadAll() to load all collections, or pass a collectionName. File: " + resourcePath); + } + + insertIntoCollection(resolvedCollectionName, dataValue, dropCollection); + } catch (IllegalArgumentException e) { + throw e; + } catch (Exception e) { + throw new RuntimeException("Failed to load test data from: " + resourcePath, e); + } + } + + public void loadAll(String resourcePath) { + loadAll(resourcePath, false); + } + + public void loadAll(String resourcePath, boolean dropCollection) { + try { + BsonDocument bsonRoot = readBsonRoot(resourcePath); + + for (Map.Entry entry : bsonRoot.entrySet()) { + insertIntoCollection(entry.getKey(), entry.getValue(), dropCollection); + } + + } catch (Exception e) { + throw new RuntimeException("Failed to load test data from: " + resourcePath, e); + } + } + private BsonDocument readBsonRoot(String resourcePath) throws Exception { + Resource resource = resourceLoader.getResource("classpath:" + resourcePath); + String content = new String(resource.getInputStream().readAllBytes(), StandardCharsets.UTF_8); + + // Normalize timezone format: +0000 -> +00:00 for BSON date parsing + content = content.replaceAll("(\\+\\d{2})(\\d{2})\"\\)", "$1:$2\")"); + + return BsonDocument.parse(content); + } + + private void insertIntoCollection(String collectionName, BsonValue dataValue, boolean dropCollection) { + List bsonDocs = new ArrayList<>(); + if (dataValue.isArray()) { + for (BsonValue item : dataValue.asArray()) { + if (item.isDocument()) { + bsonDocs.add(item.asDocument()); + } + } + } else if (dataValue.isDocument()) { + bsonDocs.add(dataValue.asDocument()); + } + + MongoCollection collection = mongoTemplate.getDb().getCollection(collectionName, BsonDocument.class); + + if (dropCollection) { + collection.deleteMany(new BsonDocument()); + } + if (!bsonDocs.isEmpty()) { + collection.insertMany(bsonDocs); + } + } +} \ No newline at end of file diff --git a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/utils/PipelineTemporaryFolderUtil.java b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/utils/PipelineTemporaryFolderUtil.java new file mode 100644 index 000000000..b1e2014bd --- /dev/null +++ b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/utils/PipelineTemporaryFolderUtil.java @@ -0,0 +1,107 @@ +package uk.ac.ebi.eva.accession.core.utils; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.io.UncheckedIOException; +import java.io.Writer; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Comparator; +import java.util.zip.GZIPOutputStream; + +public class PipelineTemporaryFolderUtil { + + private static final Logger logger = LoggerFactory.getLogger(PipelineTemporaryFolderUtil.class); + + private final Path root; + + public PipelineTemporaryFolderUtil() { + try { + this.root = Files.createTempDirectory("pipeline-test-"); + this.root.toFile().deleteOnExit(); + logger.debug("temporary folder path '{}'", root.toAbsolutePath()); + } catch (IOException e) { + throw new UncheckedIOException("Failed to create temporary folder", e); + } + } + + public File getRoot() { + return root.toFile(); + } + + public File newFolder() throws IOException { + File folder = Files.createTempDirectory(root, "tmp-dir-").toFile(); + folder.deleteOnExit(); + return folder; + } + + public File newFolder(String name) throws IOException { + File folder = root.resolve(name).toFile(); + if (!folder.exists() && !folder.mkdirs()) { + throw new IOException("Could not create directory: " + folder); + } + folder.deleteOnExit(); + return folder; + } + + public File newFile() throws IOException { + File tempFile = Files.createTempFile(root, "tmp-", null).toFile(); + tempFile.deleteOnExit(); + return tempFile; + } + + public File newFile(String name) throws IOException { + File tempFile = root.resolve(name).toFile(); + tempFile.getParentFile().mkdirs(); + if (!tempFile.exists() && !tempFile.createNewFile()) { + throw new IOException("Could not create file: " + tempFile); + } + tempFile.deleteOnExit(); + return tempFile; + } + + public File newGzipFile(String content) throws IOException { + return newGzipFile(content, null); + } + + public File newGzipFile(String content, String name) throws IOException { + File tempFile = (name == null || name.isBlank()) + ? Files.createTempFile(root, "tmp-", ".gz").toFile() + : root.resolve(name).toFile(); + + tempFile.deleteOnExit(); + + try (FileOutputStream output = new FileOutputStream(tempFile); + Writer writer = new OutputStreamWriter(new GZIPOutputStream(output), StandardCharsets.UTF_8)) { + writer.write(content); + } + + return tempFile; + } + + public void clear() { + deleteRecursively(); + } + + public void deleteRecursively() { + try { + Files.walk(root) + .sorted(Comparator.reverseOrder()) + .forEach(path -> { + try { + Files.deleteIfExists(path); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + }); + } catch (IOException e) { + throw new UncheckedIOException("Failed to delete temporary folder", e); + } + } +} \ No newline at end of file diff --git a/eva-accession-core/src/test/resources/properties/rs-accession-test.properties b/eva-accession-core/src/test/resources/properties/rs-accession-test.properties index 47f244c12..abbb7c21e 100644 --- a/eva-accession-core/src/test/resources/properties/rs-accession-test.properties +++ b/eva-accession-core/src/test/resources/properties/rs-accession-test.properties @@ -5,9 +5,6 @@ accessioning.monotonic.test-rs.blockStartValue=3000000000 accessioning.monotonic.test-rs.nextBlockInterval=1000000000 spring.data.mongodb.database=clustered-variants-test -spring.data.mongodb.host=|eva.mongo.host.test| -spring.data.mongodb.password= -spring.data.mongodb.port=27017 mongodb.read-preference=primary # See https://github.com/spring-projects/spring-boot/wiki/Spring-Boot-2.1-Release-Notes#bean-overriding diff --git a/eva-accession-core/src/test/resources/properties/ss-accession-test.properties b/eva-accession-core/src/test/resources/properties/ss-accession-test.properties index 0581ac1e4..b4151457a 100644 --- a/eva-accession-core/src/test/resources/properties/ss-accession-test.properties +++ b/eva-accession-core/src/test/resources/properties/ss-accession-test.properties @@ -13,15 +13,8 @@ spring.data.mongodb.database=submitted-variants-test # this will be replaced with a default value "localhost" when run locally # (due to the filtering attribute specified in this project's POM) # (OR) this variable can be overridden in Gitlab with "mongo" (Gitlab services are exposed under their own hostname) -spring.data.mongodb.host=|eva.mongo.host.test| -spring.data.mongodb.password= -spring.data.mongodb.port=27017 # See https://github.com/spring-projects/spring-boot/wiki/Spring-Boot-2.1-Release-Notes#bean-overriding spring.main.allow-bean-definition-overriding=true human.mongodb.database=human-variants-test -human.mongodb.host=|eva.mongo.host.test| -human.mongodb.password= -human.mongodb.port=27017 - mongodb.read-preference=primary diff --git a/eva-accession-core/src/test/resources/properties/ss-deprecation-test.properties b/eva-accession-core/src/test/resources/properties/ss-deprecation-test.properties index 22b096278..50d8e3cd5 100644 --- a/eva-accession-core/src/test/resources/properties/ss-deprecation-test.properties +++ b/eva-accession-core/src/test/resources/properties/ss-deprecation-test.properties @@ -14,15 +14,8 @@ spring.data.mongodb.database=sve-deprecation-test # this will be replaced with a default value "localhost" when run locally # (due to the filtering attribute specified in this project's POM) # (OR) this variable can be overridden in Gitlab with "mongo" (Gitlab services are exposed under their own hostname) -spring.data.mongodb.host=|eva.mongo.host.test| -spring.data.mongodb.password= -spring.data.mongodb.port=27017 # See https://github.com/spring-projects/spring-boot/wiki/Spring-Boot-2.1-Release-Notes#bean-overriding spring.main.allow-bean-definition-overriding=true human.mongodb.database=human-variants-test -human.mongodb.host=|eva.mongo.host.test| -human.mongodb.password= -human.mongodb.port=27017 - mongodb.read-preference=primary diff --git a/eva-accession-core/src/test/resources/properties/test-model.properties b/eva-accession-core/src/test/resources/properties/test-model.properties index ed3044e11..224b21c49 100644 --- a/eva-accession-core/src/test/resources/properties/test-model.properties +++ b/eva-accession-core/src/test/resources/properties/test-model.properties @@ -1,5 +1,2 @@ spring.data.mongodb.database=test-db -spring.data.mongodb.host=|eva.mongo.host.test| -spring.data.mongodb.password= -spring.data.mongodb.port=27017 mongodb.read-preference=primary \ No newline at end of file diff --git a/eva-accession-core/src/test/resources/properties/test-variants-writer.properties b/eva-accession-core/src/test/resources/properties/test-variants-writer.properties index 1d86196e5..ebf63e888 100644 --- a/eva-accession-core/src/test/resources/properties/test-variants-writer.properties +++ b/eva-accession-core/src/test/resources/properties/test-variants-writer.properties @@ -6,10 +6,6 @@ accessioning.monotonic.test-ss.blockStartValue=5000000000 accessioning.monotonic.test-ss.nextBlockInterval=1000000000 spring.data.mongodb.database=submitted-variants-test -spring.data.mongodb.host=|eva.mongo.host.test| -spring.data.mongodb.password= -spring.data.mongodb.port=27017 - mongodb.read-preference=primary # See https://github.com/spring-projects/spring-boot/wiki/Spring-Boot-2.1-Release-Notes#bean-overriding diff --git a/eva-accession-deprecate/pom.xml b/eva-accession-deprecate/pom.xml index 076bf0d58..71cf08411 100644 --- a/eva-accession-deprecate/pom.xml +++ b/eva-accession-deprecate/pom.xml @@ -2,12 +2,13 @@ + 4.0.0 + eva-accession uk.ac.ebi.eva - 0.6.58-SNAPSHOT + ${revision} - 4.0.0 eva-accession-deprecate jar @@ -28,8 +29,26 @@ - com.lordofthejars - nosqlunit-mongodb + uk.ac.ebi.eva + eva-accession-core + ${revision} + test-jar + test + + + + org.springframework.boot + spring-boot-testcontainers + test + + + org.testcontainers + mongodb + test + + + org.testcontainers + junit-jupiter test @@ -74,7 +93,4 @@ - - localhost - \ No newline at end of file diff --git a/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/batch/io/StudySubmittedVariantsReaderConfiguration.java b/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/batch/io/StudySubmittedVariantsReaderConfiguration.java index 495279961..ee471da1a 100644 --- a/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/batch/io/StudySubmittedVariantsReaderConfiguration.java +++ b/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/batch/io/StudySubmittedVariantsReaderConfiguration.java @@ -20,10 +20,9 @@ import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Import; import org.springframework.data.mongodb.core.MongoTemplate; - import uk.ac.ebi.eva.accession.core.configuration.nonhuman.MongoConfiguration; -import uk.ac.ebi.eva.accession.deprecate.configuration.BeanNames; import uk.ac.ebi.eva.accession.deprecate.batch.io.StudySubmittedVariantsReader; +import uk.ac.ebi.eva.accession.deprecate.configuration.BeanNames; import uk.ac.ebi.eva.accession.deprecate.parameters.InputParameters; @Configuration diff --git a/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/batch/jobs/DeprecateStudySubmittedVariantsJobConfiguration.java b/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/batch/jobs/DeprecateStudySubmittedVariantsJobConfiguration.java index 40d57d6d2..ebe7fa92b 100644 --- a/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/batch/jobs/DeprecateStudySubmittedVariantsJobConfiguration.java +++ b/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/batch/jobs/DeprecateStudySubmittedVariantsJobConfiguration.java @@ -17,20 +17,15 @@ import org.springframework.batch.core.Job; import org.springframework.batch.core.Step; -import org.springframework.batch.core.configuration.annotation.BatchConfigurer; import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; -import org.springframework.batch.core.configuration.annotation.JobBuilderFactory; +import org.springframework.batch.core.job.builder.JobBuilder; import org.springframework.batch.core.launch.support.RunIdIncrementer; +import org.springframework.batch.core.repository.JobRepository; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; - import uk.ac.ebi.eva.accession.deprecate.configuration.BeanNames; -import uk.ac.ebi.eva.commons.batch.configuration.SpringBoot1CompatibilityConfiguration; - -import javax.persistence.EntityManagerFactory; -import javax.sql.DataSource; @Configuration @EnableBatchProcessing @@ -41,17 +36,10 @@ public class DeprecateStudySubmittedVariantsJobConfiguration { private Step deprecateStudySubmittedVariantsStep; @Bean(BeanNames.DEPRECATE_STUDY_SUBMITTED_VARIANTS_JOB) - public Job accessionReleaseJob(JobBuilderFactory jobBuilderFactory) { - return jobBuilderFactory.get(BeanNames.DEPRECATE_STUDY_SUBMITTED_VARIANTS_JOB) - .incrementer(new RunIdIncrementer()) - .start(deprecateStudySubmittedVariantsStep) - .build(); - } - - @Bean - public BatchConfigurer configurer(DataSource dataSource, EntityManagerFactory entityManagerFactory) - throws Exception { - return SpringBoot1CompatibilityConfiguration.getSpringBoot1CompatibleBatchConfigurer(dataSource, - entityManagerFactory); + public Job accessionReleaseJob(JobRepository jobRepository) { + return new JobBuilder(BeanNames.DEPRECATE_STUDY_SUBMITTED_VARIANTS_JOB, jobRepository) + .incrementer(new RunIdIncrementer()) + .start(deprecateStudySubmittedVariantsStep) + .build(); } } diff --git a/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/batch/jobs/DeprecateSubmittedVariantsFromFileJobConfiguration.java b/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/batch/jobs/DeprecateSubmittedVariantsFromFileJobConfiguration.java index 95d9cb747..05958a588 100644 --- a/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/batch/jobs/DeprecateSubmittedVariantsFromFileJobConfiguration.java +++ b/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/batch/jobs/DeprecateSubmittedVariantsFromFileJobConfiguration.java @@ -17,19 +17,15 @@ import org.springframework.batch.core.Job; import org.springframework.batch.core.Step; -import org.springframework.batch.core.configuration.annotation.BatchConfigurer; import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; -import org.springframework.batch.core.configuration.annotation.JobBuilderFactory; +import org.springframework.batch.core.job.builder.JobBuilder; import org.springframework.batch.core.launch.support.RunIdIncrementer; +import org.springframework.batch.core.repository.JobRepository; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import uk.ac.ebi.eva.accession.deprecate.configuration.BeanNames; -import uk.ac.ebi.eva.commons.batch.configuration.SpringBoot1CompatibilityConfiguration; - -import javax.persistence.EntityManagerFactory; -import javax.sql.DataSource; @Configuration @EnableBatchProcessing @@ -40,17 +36,10 @@ public class DeprecateSubmittedVariantsFromFileJobConfiguration { private Step deprecateSubmittedVariantsFromFileStep; @Bean(BeanNames.DEPRECATE_SUBMITTED_VARIANTS_FROM_FILE_JOB) - public Job deprecateStudySubmittedVariantsFromFileJob(JobBuilderFactory jobBuilderFactory) { - return jobBuilderFactory.get(BeanNames.DEPRECATE_SUBMITTED_VARIANTS_FROM_FILE_JOB) + public Job deprecateStudySubmittedVariantsFromFileJob(JobRepository jobRepository) { + return new JobBuilder(BeanNames.DEPRECATE_SUBMITTED_VARIANTS_FROM_FILE_JOB, jobRepository) .incrementer(new RunIdIncrementer()) .start(deprecateSubmittedVariantsFromFileStep) .build(); } - - @Bean - public BatchConfigurer configurer(DataSource dataSource, EntityManagerFactory entityManagerFactory) - throws Exception { - return SpringBoot1CompatibilityConfiguration.getSpringBoot1CompatibleBatchConfigurer(dataSource, - entityManagerFactory); - } } diff --git a/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/batch/listeners/ListenerConfiguration.java b/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/batch/listeners/ListenerConfiguration.java index b0b3774f6..807e8a77e 100644 --- a/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/batch/listeners/ListenerConfiguration.java +++ b/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/batch/listeners/ListenerConfiguration.java @@ -21,14 +21,13 @@ import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Import; import org.springframework.web.client.RestTemplate; - import uk.ac.ebi.eva.accession.clustering.metric.ClusteringMetricCompute; -import uk.ac.ebi.eva.accession.deprecate.parameters.InputParameters; import uk.ac.ebi.eva.accession.core.batch.io.SubmittedVariantDeprecationWriter; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; -import uk.ac.ebi.eva.accession.deprecate.configuration.BeanNames; import uk.ac.ebi.eva.accession.deprecate.batch.listeners.DeprecationStepProgressListener; +import uk.ac.ebi.eva.accession.deprecate.configuration.BeanNames; import uk.ac.ebi.eva.accession.deprecate.configuration.InputParametersConfiguration; +import uk.ac.ebi.eva.accession.deprecate.parameters.InputParameters; import uk.ac.ebi.eva.metrics.configuration.MetricConfiguration; import uk.ac.ebi.eva.metrics.count.CountServiceParameters; import uk.ac.ebi.eva.metrics.metric.MetricCompute; diff --git a/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/batch/steps/DeprecateStudySubmittedVariantsStepConfiguration.java b/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/batch/steps/DeprecateStudySubmittedVariantsStepConfiguration.java index 86eaed6d8..37261c120 100644 --- a/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/batch/steps/DeprecateStudySubmittedVariantsStepConfiguration.java +++ b/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/batch/steps/DeprecateStudySubmittedVariantsStepConfiguration.java @@ -18,7 +18,8 @@ import org.springframework.batch.core.Step; import org.springframework.batch.core.StepExecutionListener; import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; -import org.springframework.batch.core.configuration.annotation.StepBuilderFactory; +import org.springframework.batch.core.repository.JobRepository; +import org.springframework.batch.core.step.builder.StepBuilder; import org.springframework.batch.core.step.tasklet.TaskletStep; import org.springframework.batch.item.ItemStreamReader; import org.springframework.batch.item.ItemWriter; @@ -27,7 +28,7 @@ import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; - +import org.springframework.transaction.PlatformTransactionManager; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; import uk.ac.ebi.eva.accession.deprecate.configuration.BeanNames; @@ -48,10 +49,11 @@ public class DeprecateStudySubmittedVariantsStepConfiguration { private StepExecutionListener progressListener; @Bean(BeanNames.DEPRECATE_STUDY_SUBMITTED_VARIANTS_STEP) - public Step deprecateClusteredVariantsStep(StepBuilderFactory stepBuilderFactory, + public Step deprecateClusteredVariantsStep(JobRepository jobRepository, + PlatformTransactionManager transactionManager, SimpleCompletionPolicy chunkSizeCompletionPolicy) { - TaskletStep step = stepBuilderFactory.get(BeanNames.DEPRECATE_STUDY_SUBMITTED_VARIANTS_STEP) - .chunk(chunkSizeCompletionPolicy) + TaskletStep step = new StepBuilder(BeanNames.DEPRECATE_STUDY_SUBMITTED_VARIANTS_STEP, jobRepository) + .chunk(chunkSizeCompletionPolicy, transactionManager) .reader(studySubmittedVariantsReader) .writer(submittedVariantDeprecationWriter) .listener(progressListener) diff --git a/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/batch/steps/DeprecateSubmittedVariantsFromFileStepConfiguration.java b/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/batch/steps/DeprecateSubmittedVariantsFromFileStepConfiguration.java index 1ef5835fa..aad2dfedd 100644 --- a/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/batch/steps/DeprecateSubmittedVariantsFromFileStepConfiguration.java +++ b/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/batch/steps/DeprecateSubmittedVariantsFromFileStepConfiguration.java @@ -18,7 +18,8 @@ import org.springframework.batch.core.Step; import org.springframework.batch.core.StepExecutionListener; import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; -import org.springframework.batch.core.configuration.annotation.StepBuilderFactory; +import org.springframework.batch.core.repository.JobRepository; +import org.springframework.batch.core.step.builder.StepBuilder; import org.springframework.batch.core.step.tasklet.TaskletStep; import org.springframework.batch.item.ItemStreamReader; import org.springframework.batch.item.ItemWriter; @@ -27,6 +28,7 @@ import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.transaction.PlatformTransactionManager; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; import uk.ac.ebi.eva.accession.deprecate.configuration.BeanNames; @@ -47,10 +49,11 @@ public class DeprecateSubmittedVariantsFromFileStepConfiguration { private StepExecutionListener progressListener; @Bean(BeanNames.DEPRECATE_SUBMITTED_VARIANTS_FROM_FILE_STEP) - public Step deprecateSubmittedVariantsFromFileStep(StepBuilderFactory stepBuilderFactory, - SimpleCompletionPolicy chunkSizeCompletionPolicy) { - TaskletStep step = stepBuilderFactory.get(BeanNames.DEPRECATE_SUBMITTED_VARIANTS_FROM_FILE_STEP) - .chunk(chunkSizeCompletionPolicy) + public Step deprecateSubmittedVariantsFromFileStep(JobRepository jobRepository, + PlatformTransactionManager transactionManager, + SimpleCompletionPolicy chunkSizeCompletionPolicy) { + TaskletStep step = new StepBuilder(BeanNames.DEPRECATE_SUBMITTED_VARIANTS_FROM_FILE_STEP, jobRepository) + .chunk(chunkSizeCompletionPolicy, transactionManager) .reader(submittedVariantsFileReader) .writer(submittedVariantDeprecationWriter) .listener(progressListener) diff --git a/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/MongoTestDatabaseSetup.java b/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/MongoTestDatabaseSetup.java index 56efe9658..3488818f1 100644 --- a/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/MongoTestDatabaseSetup.java +++ b/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/MongoTestDatabaseSetup.java @@ -32,8 +32,8 @@ import java.util.Arrays; import java.util.function.Function; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; public class MongoTestDatabaseSetup { diff --git a/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/batch/io/StudySubmittedVariantsReaderTest.java b/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/batch/io/StudySubmittedVariantsReaderTest.java index 29eb3e093..ea6b98198 100644 --- a/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/batch/io/StudySubmittedVariantsReaderTest.java +++ b/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/batch/io/StudySubmittedVariantsReaderTest.java @@ -15,43 +15,34 @@ */ package uk.ac.ebi.eva.accession.deprecate.batch.io; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; -import com.mongodb.MongoClient; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.batch.item.ExecutionContext; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.ApplicationContext; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; - +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.MongoConfiguration; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.deprecate.test.configuration.MongoTestConfiguration; -import uk.ac.ebi.eva.accession.deprecate.test.rule.FixSpringMongoDbRule; import java.util.ArrayList; import java.util.List; import static junit.framework.TestCase.assertTrue; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import static uk.ac.ebi.eva.accession.deprecate.MongoTestDatabaseSetup.ASSEMBLY; -import static uk.ac.ebi.eva.accession.deprecate.MongoTestDatabaseSetup.populateTestDB; import static uk.ac.ebi.eva.accession.deprecate.MongoTestDatabaseSetup.STUDY1; +import static uk.ac.ebi.eva.accession.deprecate.MongoTestDatabaseSetup.populateTestDB; -@RunWith(SpringRunner.class) +@ExtendWith(SpringExtension.class) @TestPropertySource("classpath:study-submitted-variants-test.properties") @ContextConfiguration(classes = {MongoConfiguration.class, MongoTestConfiguration.class}) -public class StudySubmittedVariantsReaderTest { - - private static final String TEST_DB = "test-db"; - +public class StudySubmittedVariantsReaderTest extends MongoTestContainerHelper { private static final String ID_1 = "hash5"; private static final String ID_2 = "hash6"; @@ -64,38 +55,28 @@ public class StudySubmittedVariantsReaderTest { private StudySubmittedVariantsReader reader; - @Autowired - private MongoClient mongoClient; - @Autowired private MongoTemplate mongoTemplate; - //Required by nosql-unit - @Autowired - private ApplicationContext applicationContext; - - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName(TEST_DB).build()); - - @Before + @BeforeEach public void setUp() { + mongoTemplate.getDb().drop(); populateTestDB(this.mongoTemplate); executionContext = new ExecutionContext(); reader = new StudySubmittedVariantsReader(ASSEMBLY, STUDY1, mongoTemplate, CHUNK_SIZE); reader.open(executionContext); } - @After + @AfterEach public void tearDown() { + mongoTemplate.getDb().drop(); reader.close(); - mongoClient.dropDatabase(TEST_DB); } @Test public void readStudySubmittedVariants() { List variants = readIntoList(); - assertEquals(3, variants.size()); + assertEquals(3, variants.size()); assertTrue(variants.stream().anyMatch(x -> x.getId().equals(ID_1))); assertTrue(variants.stream().anyMatch(x -> x.getId().equals(ID_2))); assertTrue(variants.stream().anyMatch(x -> x.getId().equals(ID_3))); diff --git a/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/configuration/jobs/DeprecateStudySubmittedVariantsJobConfigurationTest.java b/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/configuration/jobs/DeprecateStudySubmittedVariantsJobConfigurationTest.java index 01ed86b5f..2c44eb795 100644 --- a/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/configuration/jobs/DeprecateStudySubmittedVariantsJobConfigurationTest.java +++ b/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/configuration/jobs/DeprecateStudySubmittedVariantsJobConfigurationTest.java @@ -15,14 +15,10 @@ */ package uk.ac.ebi.eva.accession.deprecate.configuration.jobs; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; -import com.mongodb.MongoClient; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.batch.core.BatchStatus; import org.springframework.batch.core.JobExecution; import org.springframework.batch.core.StepExecution; @@ -32,37 +28,34 @@ import org.springframework.batch.test.JobRepositoryTestUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.context.ApplicationContext; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.eva.accession.core.runner.CommandLineRunnerUtils; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.deprecate.MongoTestDatabaseSetup; +import uk.ac.ebi.eva.accession.deprecate.test.configuration.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.accession.deprecate.test.configuration.BatchTestConfiguration; import uk.ac.ebi.eva.accession.deprecate.test.configuration.MongoTestConfiguration; -import uk.ac.ebi.eva.accession.deprecate.test.rule.FixSpringMongoDbRule; -import javax.sql.DataSource; import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.stream.Collectors; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; import static uk.ac.ebi.eva.accession.deprecate.configuration.BeanNames.DEPRECATE_STUDY_SUBMITTED_VARIANTS_JOB; import static uk.ac.ebi.eva.accession.deprecate.configuration.BeanNames.DEPRECATE_STUDY_SUBMITTED_VARIANTS_STEP; import static uk.ac.ebi.eva.accession.deprecate.test.configuration.BatchTestConfiguration.JOB_LAUNCHER_FROM_MONGO; -@RunWith(SpringRunner.class) -@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class}) +@ExtendWith(SpringExtension.class) +@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class, + BatchJobRepositoryTestConfiguration.class}) @TestPropertySource("classpath:study-submitted-variants-test.properties") -public class DeprecateStudySubmittedVariantsJobConfigurationTest { - - private static final String TEST_DB = "test-db"; - +public class DeprecateStudySubmittedVariantsJobConfigurationTest extends MongoTestContainerHelper { @Autowired @Qualifier(JOB_LAUNCHER_FROM_MONGO) private JobLauncherTestUtils jobLauncherTestUtilsFromMongo; @@ -73,36 +66,22 @@ public class DeprecateStudySubmittedVariantsJobConfigurationTest { @Autowired private JobRepository jobRepository; - @Autowired - private DataSource datasource; - - @Autowired - private MongoClient mongoClient; - @Autowired private MongoTemplate mongoTemplate; private JobRepositoryTestUtils jobRepositoryTestUtils; - //Required by nosql-unit - @Autowired - private ApplicationContext applicationContext; - - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName(TEST_DB).build()); - - @Before - public void setUp() throws Exception { - this.mongoClient.dropDatabase(TEST_DB); + @BeforeEach + public void setUp() { + mongoTemplate.getDb().drop(); MongoTestDatabaseSetup.populateTestDB(this.mongoTemplate); - jobRepositoryTestUtils = new JobRepositoryTestUtils(jobRepository, datasource); + jobRepositoryTestUtils = new JobRepositoryTestUtils(jobRepository); jobRepositoryTestUtils.removeJobExecutions(); } - @After + @AfterEach public void tearDown() { - this.mongoClient.dropDatabase(TEST_DB); + mongoTemplate.getDb().drop(); jobRepositoryTestUtils.removeJobExecutions(); } @@ -134,18 +113,18 @@ public void restartCompletedJobThatIsAlreadyInTheRepository() throws Exception { MongoTestDatabaseSetup.assertPostDeprecationDatabaseState(this.mongoTemplate); long instanceIdFirstJob = CommandLineRunnerUtils.getLastJobExecution(DEPRECATE_STUDY_SUBMITTED_VARIANTS_JOB, - jobExplorer, - jobExecution.getJobParameters()) - .getJobInstance().getInstanceId(); + jobExplorer, + jobExecution.getJobParameters()) + .getJobInstance().getInstanceId(); jobExecution = jobLauncherTestUtilsFromMongo.launchJob(); expectedSteps = Collections.singletonList(DEPRECATE_STUDY_SUBMITTED_VARIANTS_STEP); assertStepsExecuted(expectedSteps, jobExecution); assertEquals(BatchStatus.COMPLETED, jobExecution.getStatus()); long instanceIdSecondJob = CommandLineRunnerUtils.getLastJobExecution(DEPRECATE_STUDY_SUBMITTED_VARIANTS_JOB, - jobExplorer, - jobExecution.getJobParameters()) - .getJobInstance().getInstanceId(); + jobExplorer, + jobExecution.getJobParameters()) + .getJobInstance().getInstanceId(); assertNotEquals(instanceIdSecondJob, instanceIdFirstJob); MongoTestDatabaseSetup.assertPostDeprecationDatabaseState(this.mongoTemplate); } diff --git a/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/configuration/jobs/DeprecateSubmittedVariantsFromFileJobConfigurationTest.java b/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/configuration/jobs/DeprecateSubmittedVariantsFromFileJobConfigurationTest.java index 796759f8f..693ff64f1 100644 --- a/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/configuration/jobs/DeprecateSubmittedVariantsFromFileJobConfigurationTest.java +++ b/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/configuration/jobs/DeprecateSubmittedVariantsFromFileJobConfigurationTest.java @@ -15,14 +15,10 @@ */ package uk.ac.ebi.eva.accession.deprecate.configuration.jobs; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; -import com.mongodb.MongoClient; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.batch.core.BatchStatus; import org.springframework.batch.core.JobExecution; import org.springframework.batch.core.StepExecution; @@ -32,37 +28,34 @@ import org.springframework.batch.test.JobRepositoryTestUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.context.ApplicationContext; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.eva.accession.core.runner.CommandLineRunnerUtils; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.deprecate.MongoTestDatabaseSetup; +import uk.ac.ebi.eva.accession.deprecate.test.configuration.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.accession.deprecate.test.configuration.BatchTestConfiguration; import uk.ac.ebi.eva.accession.deprecate.test.configuration.MongoTestConfiguration; -import uk.ac.ebi.eva.accession.deprecate.test.rule.FixSpringMongoDbRule; -import javax.sql.DataSource; import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.stream.Collectors; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; import static uk.ac.ebi.eva.accession.deprecate.configuration.BeanNames.DEPRECATE_SUBMITTED_VARIANTS_FROM_FILE_JOB; import static uk.ac.ebi.eva.accession.deprecate.configuration.BeanNames.DEPRECATE_SUBMITTED_VARIANTS_FROM_FILE_STEP; import static uk.ac.ebi.eva.accession.deprecate.test.configuration.BatchTestConfiguration.JOB_LAUNCHER_FROM_FILE; -@RunWith(SpringRunner.class) -@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class}) +@ExtendWith(SpringExtension.class) +@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class, + BatchJobRepositoryTestConfiguration.class}) @TestPropertySource("classpath:deprecate-submitted-variants-from-file-test.properties") -public class DeprecateSubmittedVariantsFromFileJobConfigurationTest { - - private static final String TEST_DB = "test-db"; - +public class DeprecateSubmittedVariantsFromFileJobConfigurationTest extends MongoTestContainerHelper { @Autowired @Qualifier(JOB_LAUNCHER_FROM_FILE) private JobLauncherTestUtils jobLauncherTestUtilsFromFile; @@ -73,36 +66,22 @@ public class DeprecateSubmittedVariantsFromFileJobConfigurationTest { @Autowired private JobRepository jobRepository; - @Autowired - private DataSource datasource; - - @Autowired - private MongoClient mongoClient; - @Autowired private MongoTemplate mongoTemplate; private JobRepositoryTestUtils jobRepositoryTestUtils; - //Required by nosql-unit - @Autowired - private ApplicationContext applicationContext; - - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName(TEST_DB).build()); - - @Before - public void setUp() throws Exception { - this.mongoClient.dropDatabase(TEST_DB); + @BeforeEach + public void setUp() { + mongoTemplate.getDb().drop(); MongoTestDatabaseSetup.populateTestDBForFile(this.mongoTemplate); - jobRepositoryTestUtils = new JobRepositoryTestUtils(jobRepository, datasource); + jobRepositoryTestUtils = new JobRepositoryTestUtils(jobRepository); jobRepositoryTestUtils.removeJobExecutions(); } - @After + @AfterEach public void tearDown() { - this.mongoClient.dropDatabase(TEST_DB); + mongoTemplate.getDb().drop(); jobRepositoryTestUtils.removeJobExecutions(); } diff --git a/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/configuration/steps/DeprecateStudySubmittedVariantsStepConfigurationTest.java b/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/configuration/steps/DeprecateStudySubmittedVariantsStepConfigurationTest.java index 469370fc5..67ad9f95a 100644 --- a/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/configuration/steps/DeprecateStudySubmittedVariantsStepConfigurationTest.java +++ b/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/configuration/steps/DeprecateStudySubmittedVariantsStepConfigurationTest.java @@ -15,68 +15,50 @@ */ package uk.ac.ebi.eva.accession.deprecate.configuration.steps; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; -import com.mongodb.MongoClient; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.batch.core.BatchStatus; import org.springframework.batch.core.JobExecution; import org.springframework.batch.test.JobLauncherTestUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.context.ApplicationContext; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; - +import org.springframework.test.context.junit.jupiter.SpringExtension; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.deprecate.MongoTestDatabaseSetup; import uk.ac.ebi.eva.accession.deprecate.configuration.BeanNames; +import uk.ac.ebi.eva.accession.deprecate.test.configuration.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.accession.deprecate.test.configuration.BatchTestConfiguration; import uk.ac.ebi.eva.accession.deprecate.test.configuration.MongoTestConfiguration; -import uk.ac.ebi.eva.accession.deprecate.test.rule.FixSpringMongoDbRule; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import static uk.ac.ebi.eva.accession.deprecate.test.configuration.BatchTestConfiguration.JOB_LAUNCHER_FROM_MONGO; -@RunWith(SpringRunner.class) -@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class}) +@ExtendWith(SpringExtension.class) +@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class, + BatchJobRepositoryTestConfiguration.class}) @TestPropertySource("classpath:study-submitted-variants-test.properties") -public class DeprecateStudySubmittedVariantsStepConfigurationTest { - - private static final String TEST_DB = "test-db"; - +public class DeprecateStudySubmittedVariantsStepConfigurationTest extends MongoTestContainerHelper { @Autowired @Qualifier(JOB_LAUNCHER_FROM_MONGO) private JobLauncherTestUtils jobLauncherTestUtilsFromMongo; - @Autowired - private MongoClient mongoClient; - @Autowired private MongoTemplate mongoTemplate; - //Required by nosql-unit - @Autowired - private ApplicationContext applicationContext; - - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName(TEST_DB).build()); - - @Before + @BeforeEach public void setUp() { - this.mongoClient.dropDatabase(TEST_DB); + mongoTemplate.getDb().drop(); MongoTestDatabaseSetup.populateTestDB(this.mongoTemplate); } - @After + @AfterEach public void tearDown() { - this.mongoClient.dropDatabase(TEST_DB); + mongoTemplate.getDb().drop(); } @Test diff --git a/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/configuration/steps/DeprecateSubmittedVariantsFromFileStepConfigurationTest.java b/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/configuration/steps/DeprecateSubmittedVariantsFromFileStepConfigurationTest.java index 7f319abeb..05ab6f356 100644 --- a/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/configuration/steps/DeprecateSubmittedVariantsFromFileStepConfigurationTest.java +++ b/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/configuration/steps/DeprecateSubmittedVariantsFromFileStepConfigurationTest.java @@ -15,67 +15,50 @@ */ package uk.ac.ebi.eva.accession.deprecate.configuration.steps; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; -import com.mongodb.MongoClient; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.batch.core.BatchStatus; import org.springframework.batch.core.JobExecution; import org.springframework.batch.test.JobLauncherTestUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.context.ApplicationContext; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.deprecate.MongoTestDatabaseSetup; import uk.ac.ebi.eva.accession.deprecate.configuration.BeanNames; +import uk.ac.ebi.eva.accession.deprecate.test.configuration.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.accession.deprecate.test.configuration.BatchTestConfiguration; import uk.ac.ebi.eva.accession.deprecate.test.configuration.MongoTestConfiguration; -import uk.ac.ebi.eva.accession.deprecate.test.rule.FixSpringMongoDbRule; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import static uk.ac.ebi.eva.accession.deprecate.test.configuration.BatchTestConfiguration.JOB_LAUNCHER_FROM_FILE; -@RunWith(SpringRunner.class) -@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class}) +@ExtendWith(SpringExtension.class) +@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class, + BatchJobRepositoryTestConfiguration.class}) @TestPropertySource("classpath:deprecate-submitted-variants-from-file-test.properties") -public class DeprecateSubmittedVariantsFromFileStepConfigurationTest { - - private static final String TEST_DB = "test-db"; - +public class DeprecateSubmittedVariantsFromFileStepConfigurationTest extends MongoTestContainerHelper { @Autowired @Qualifier(JOB_LAUNCHER_FROM_FILE) private JobLauncherTestUtils jobLauncherTestUtilsFromFile; - @Autowired - private MongoClient mongoClient; - @Autowired private MongoTemplate mongoTemplate; - //Required by nosql-unit - @Autowired - private ApplicationContext applicationContext; - - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName(TEST_DB).build()); - - @Before + @BeforeEach public void setUp() { - this.mongoClient.dropDatabase(TEST_DB); + mongoTemplate.getDb().drop(); MongoTestDatabaseSetup.populateTestDBForFile(this.mongoTemplate); } - @After + @AfterEach public void tearDown() { - this.mongoClient.dropDatabase(TEST_DB); + mongoTemplate.getDb().drop(); } @Test diff --git a/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/test/configuration/BatchJobRepositoryTestConfiguration.java b/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/test/configuration/BatchJobRepositoryTestConfiguration.java new file mode 100644 index 000000000..854541a72 --- /dev/null +++ b/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/test/configuration/BatchJobRepositoryTestConfiguration.java @@ -0,0 +1,22 @@ +package uk.ac.ebi.eva.accession.deprecate.test.configuration; + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Primary; +import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseBuilder; +import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseType; + +import javax.sql.DataSource; + +@Configuration +public class BatchJobRepositoryTestConfiguration { + + @Bean + @Primary + public DataSource dataSource() { + return new EmbeddedDatabaseBuilder() + .setType(EmbeddedDatabaseType.HSQL) + .addScript("org/springframework/batch/core/schema-hsqldb.sql") + .build(); + } +} \ No newline at end of file diff --git a/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/test/configuration/BatchTestConfiguration.java b/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/test/configuration/BatchTestConfiguration.java index 05b7ccab3..f30f6de12 100644 --- a/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/test/configuration/BatchTestConfiguration.java +++ b/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/test/configuration/BatchTestConfiguration.java @@ -16,8 +16,9 @@ package uk.ac.ebi.eva.accession.deprecate.test.configuration; import org.springframework.batch.core.Job; +import org.springframework.batch.core.launch.JobLauncher; +import org.springframework.batch.core.repository.JobRepository; import org.springframework.batch.test.JobLauncherTestUtils; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.context.annotation.Bean; @@ -27,15 +28,15 @@ import uk.ac.ebi.eva.accession.deprecate.configuration.batch.io.StudySubmittedVariantsFileReaderConfiguration; import uk.ac.ebi.eva.accession.deprecate.configuration.batch.io.StudySubmittedVariantsReaderConfiguration; import uk.ac.ebi.eva.accession.deprecate.configuration.batch.io.SubmittedVariantDeprecationWriterConfiguration; -import uk.ac.ebi.eva.accession.deprecate.configuration.batch.jobs.DeprecateSubmittedVariantsFromFileJobConfiguration; import uk.ac.ebi.eva.accession.deprecate.configuration.batch.jobs.DeprecateStudySubmittedVariantsJobConfiguration; +import uk.ac.ebi.eva.accession.deprecate.configuration.batch.jobs.DeprecateSubmittedVariantsFromFileJobConfiguration; import uk.ac.ebi.eva.accession.deprecate.configuration.batch.listeners.ListenerConfiguration; import uk.ac.ebi.eva.accession.deprecate.configuration.batch.policies.ChunkSizeCompletionPolicyConfiguration; -import uk.ac.ebi.eva.accession.deprecate.configuration.batch.steps.DeprecateSubmittedVariantsFromFileStepConfiguration; import uk.ac.ebi.eva.accession.deprecate.configuration.batch.steps.DeprecateStudySubmittedVariantsStepConfiguration; +import uk.ac.ebi.eva.accession.deprecate.configuration.batch.steps.DeprecateSubmittedVariantsFromFileStepConfiguration; -import static uk.ac.ebi.eva.accession.deprecate.configuration.BeanNames.DEPRECATE_SUBMITTED_VARIANTS_FROM_FILE_JOB; import static uk.ac.ebi.eva.accession.deprecate.configuration.BeanNames.DEPRECATE_STUDY_SUBMITTED_VARIANTS_JOB; +import static uk.ac.ebi.eva.accession.deprecate.configuration.BeanNames.DEPRECATE_SUBMITTED_VARIANTS_FROM_FILE_JOB; @EnableAutoConfiguration @Import({MongoConfiguration.class, @@ -51,31 +52,27 @@ ListenerConfiguration.class }) public class BatchTestConfiguration { - public static final String JOB_LAUNCHER_FROM_MONGO = "JOB_LAUNCHER_FROM_MONGO"; public static final String JOB_LAUNCHER_FROM_FILE = "JOB_LAUNCHER_FROM_FILE"; @Bean(JOB_LAUNCHER_FROM_MONGO) - public JobLauncherTestUtils jobLauncherTestUtilsFromMongo() { - - return new JobLauncherTestUtils() { - @Override - @Autowired - public void setJob(@Qualifier(DEPRECATE_STUDY_SUBMITTED_VARIANTS_JOB) Job job) { - super.setJob(job); - } - }; + public JobLauncherTestUtils jobLauncherTestUtilsFromMongo(JobLauncher jobLauncher, JobRepository jobRepository, + @Qualifier(DEPRECATE_STUDY_SUBMITTED_VARIANTS_JOB) Job job) { + JobLauncherTestUtils utils = new JobLauncherTestUtils(); + utils.setJobLauncher(jobLauncher); + utils.setJobRepository(jobRepository); + utils.setJob(job); + return utils; } @Bean(JOB_LAUNCHER_FROM_FILE) - public JobLauncherTestUtils jobLauncherTestUtilsFromFile() { + public JobLauncherTestUtils jobLauncherTestUtilsFromFile(JobLauncher jobLauncher, JobRepository jobRepository, + @Qualifier(DEPRECATE_SUBMITTED_VARIANTS_FROM_FILE_JOB) Job job) { + JobLauncherTestUtils utils = new JobLauncherTestUtils(); + utils.setJobLauncher(jobLauncher); + utils.setJobRepository(jobRepository); + utils.setJob(job); + return utils; - return new JobLauncherTestUtils() { - @Override - @Autowired - public void setJob(@Qualifier(DEPRECATE_SUBMITTED_VARIANTS_FROM_FILE_JOB) Job job) { - super.setJob(job); - } - }; } } diff --git a/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/test/configuration/MongoTestConfiguration.java b/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/test/configuration/MongoTestConfiguration.java index 06aa1e647..08c2a4774 100644 --- a/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/test/configuration/MongoTestConfiguration.java +++ b/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/test/configuration/MongoTestConfiguration.java @@ -17,12 +17,30 @@ */ package uk.ac.ebi.eva.accession.deprecate.test.configuration; +import com.mongodb.ReadPreference; +import com.mongodb.WriteConcern; +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoClients; +import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.domain.EntityScan; import org.springframework.boot.test.autoconfigure.data.mongo.AutoConfigureDataMongo; +import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Primary; +import org.springframework.data.mongodb.MongoDatabaseFactory; import org.springframework.data.mongodb.config.EnableMongoAuditing; +import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.data.mongodb.core.SimpleMongoClientDatabaseFactory; +import org.springframework.data.mongodb.core.WriteResultChecking; +import org.springframework.data.mongodb.core.convert.DefaultDbRefResolver; +import org.springframework.data.mongodb.core.convert.DefaultMongoTypeMapper; +import org.springframework.data.mongodb.core.convert.MappingMongoConverter; +import org.springframework.data.mongodb.core.convert.MongoCustomConversions; +import org.springframework.data.mongodb.core.mapping.MongoMappingContext; import org.springframework.data.mongodb.repository.config.EnableMongoRepositories; +import java.util.Collections; + @Configuration @EntityScan(basePackages = {"uk.ac.ebi.eva.accession.core.persistence"}) @EnableMongoRepositories(basePackages = { @@ -32,4 +50,56 @@ @AutoConfigureDataMongo public class MongoTestConfiguration { + @Value("${spring.data.mongodb.uri}") + private String mongoUri; + + @Value("${spring.data.mongodb.database}") + private String database; + + @Value("${mongodb.read-preference:primary}") + private String readPreference; + + @Bean + @Primary + public MongoClient mongoClient() { + return MongoClients.create(mongoUri); + } + + @Bean("primaryFactory") + @Primary + public MongoDatabaseFactory mongoDbFactory() { + return new SimpleMongoClientDatabaseFactory(mongoClient(), database); + } + + @Bean + @Primary + public MongoCustomConversions mongoCustomConversions() { + return new MongoCustomConversions(Collections.emptyList()); + } + + @Bean + @Primary + public MongoMappingContext mongoMappingContext(MongoCustomConversions conversions) { + MongoMappingContext context = new MongoMappingContext(); + context.setSimpleTypeHolder(conversions.getSimpleTypeHolder()); + context.setAutoIndexCreation(false); + return context; + } + + @Bean + @Primary + public MongoTemplate mongoTemplate(MongoMappingContext mongoMappingContext, + MongoCustomConversions conversions) { + MappingMongoConverter converter = new MappingMongoConverter( + new DefaultDbRefResolver(mongoDbFactory()), mongoMappingContext); + converter.setCustomConversions(conversions); + converter.setTypeMapper(new DefaultMongoTypeMapper(null)); + converter.afterPropertiesSet(); + MongoTemplate mongoTemplate = new MongoTemplate(mongoDbFactory(), converter); + mongoTemplate.setReadPreference(ReadPreference.valueOf(readPreference)); + mongoTemplate.setWriteConcern(WriteConcern.MAJORITY); + mongoTemplate.setWriteResultChecking(WriteResultChecking.EXCEPTION); + return mongoTemplate; + } + } \ No newline at end of file diff --git a/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/test/rule/FixSpringMongoDbRule.java b/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/test/rule/FixSpringMongoDbRule.java deleted file mode 100644 index a4dffb32b..000000000 --- a/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/test/rule/FixSpringMongoDbRule.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * - * Copyright 2022 EMBL - European Bioinformatics Institute - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -package uk.ac.ebi.eva.accession.deprecate.test.rule; - -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfiguration; -import com.lordofthejars.nosqlunit.mongodb.SpringMongoDbRule; - -/** - * Temporary fix until nosql unit rc-6 or final is released - * TODO Move this to the core module (used in the release and deprecate module) - */ -public class FixSpringMongoDbRule extends SpringMongoDbRule { - - public FixSpringMongoDbRule(MongoDbConfiguration mongoDbConfiguration) { - super(mongoDbConfiguration); - } - - public FixSpringMongoDbRule(MongoDbConfiguration mongoDbConfiguration, Object object) { - super(mongoDbConfiguration, object); - } - - @Override - public void close() { - // DO NOT CLOSE the connection (Spring will do it when destroying the context) - } - -} diff --git a/eva-accession-deprecate/src/test/resources/deprecate-submitted-variants-from-file-test.properties b/eva-accession-deprecate/src/test/resources/deprecate-submitted-variants-from-file-test.properties index 68229ca2e..5ea0e7638 100644 --- a/eva-accession-deprecate/src/test/resources/deprecate-submitted-variants-from-file-test.properties +++ b/eva-accession-deprecate/src/test/resources/deprecate-submitted-variants-from-file-test.properties @@ -26,15 +26,8 @@ spring.data.mongodb.database=test-db # this will be replaced with a default value "localhost" when run locally # (due to the filtering attribute specified in this project's POM) # (OR) this variable can be overridden in Gitlab with "mongo" (Gitlab services are exposed under their own hostname) -spring.data.mongodb.host=|eva.mongo.host.test| -spring.data.mongodb.password= -spring.data.mongodb.port=27017 # See https://github.com/spring-projects/spring-boot/wiki/Spring-Boot-2.1-Release-Notes#bean-overriding spring.main.allow-bean-definition-overriding=true human.mongodb.database=human-variants-test -human.mongodb.host=|eva.mongo.host.test| -human.mongodb.password= -human.mongodb.port=27017 - mongodb.read-preference=primary diff --git a/eva-accession-deprecate/src/test/resources/study-submitted-variants-test.properties b/eva-accession-deprecate/src/test/resources/study-submitted-variants-test.properties index 8dbef8788..e0e437959 100644 --- a/eva-accession-deprecate/src/test/resources/study-submitted-variants-test.properties +++ b/eva-accession-deprecate/src/test/resources/study-submitted-variants-test.properties @@ -23,15 +23,8 @@ spring.data.mongodb.database=test-db # this will be replaced with a default value "localhost" when run locally # (due to the filtering attribute specified in this project's POM) # (OR) this variable can be overridden in Gitlab with "mongo" (Gitlab services are exposed under their own hostname) -spring.data.mongodb.host=|eva.mongo.host.test| -spring.data.mongodb.password= -spring.data.mongodb.port=27017 # See https://github.com/spring-projects/spring-boot/wiki/Spring-Boot-2.1-Release-Notes#bean-overriding spring.main.allow-bean-definition-overriding=true human.mongodb.database=human-variants-test -human.mongodb.host=|eva.mongo.host.test| -human.mongodb.password= -human.mongodb.port=27017 - mongodb.read-preference=primary diff --git a/eva-accession-import-dbsnp2/pom.xml b/eva-accession-import-dbsnp2/pom.xml index 45f349d48..03889334e 100644 --- a/eva-accession-import-dbsnp2/pom.xml +++ b/eva-accession-import-dbsnp2/pom.xml @@ -2,12 +2,13 @@ + 4.0.0 + eva-accession uk.ac.ebi.eva - 0.6.58-SNAPSHOT + ${revision} - 4.0.0 eva-accession-import-dbsnp2 jar @@ -44,19 +45,28 @@ test - com.lordofthejars - nosqlunit-mongodb + org.springframework.boot + spring-boot-testcontainers + test + + + org.testcontainers + mongodb + test + + + org.testcontainers + junit-jupiter test - org.codehaus.jackson - jackson-mapper-asl - 1.9.13 + com.fasterxml.jackson.core + jackson-databind org.apache.commons commons-compress - 1.18 + 1.28.0 uk.ac.ebi.eva diff --git a/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/batch/io/DbsnpJsonClusteredVariantsWriter.java b/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/batch/io/DbsnpJsonClusteredVariantsWriter.java index 77175b2b6..1fcd0da93 100644 --- a/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/batch/io/DbsnpJsonClusteredVariantsWriter.java +++ b/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/batch/io/DbsnpJsonClusteredVariantsWriter.java @@ -18,17 +18,17 @@ import com.mongodb.MongoBulkWriteException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.batch.item.Chunk; import org.springframework.batch.item.ItemWriter; -import org.springframework.dao.DuplicateKeyException; +import org.springframework.data.mongodb.BulkOperationException; import uk.ac.ebi.ampt2d.commons.accession.core.models.EventType; - -import uk.ac.ebi.eva.accession.core.batch.io.DbsnpClusteredVariantWriter; import uk.ac.ebi.eva.accession.core.batch.io.DbsnpClusteredVariantOperationWriter; +import uk.ac.ebi.eva.accession.core.batch.io.DbsnpClusteredVariantWriter; import uk.ac.ebi.eva.accession.core.batch.io.MergeOperationBuilder; -import uk.ac.ebi.eva.accession.core.repository.nonhuman.dbsnp.DbsnpClusteredVariantAccessioningRepository; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpClusteredVariantEntity; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpClusteredVariantInactiveEntity; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpClusteredVariantOperationEntity; +import uk.ac.ebi.eva.accession.core.repository.nonhuman.dbsnp.DbsnpClusteredVariantAccessioningRepository; import uk.ac.ebi.eva.accession.core.repository.nonhuman.dbsnp.DbsnpClusteredVariantOperationRepository; import java.util.Arrays; @@ -61,21 +61,20 @@ public DbsnpJsonClusteredVariantsWriter(DbsnpClusteredVariantWriter dbsnpCluster } @Override - public void write(List clusteredVariants) throws Exception { + public void write(Chunk clusteredVariants) throws Exception { try { if (!clusteredVariants.isEmpty()) { dbsnpClusteredVariantWriter.write(clusteredVariants); - } - else { + } else { logger.warn("Could not find any clustered variants to write in the current chunk!"); } - } catch (DuplicateKeyException exception) { + } catch (BulkOperationException exception) { MongoBulkWriteException writeException = ((MongoBulkWriteException) exception.getCause()); List mergeClusteredOperations = clusteredOperationBuilder.buildMergeOperationsFromException( - (List) clusteredVariants, writeException); + List.copyOf(clusteredVariants.getItems()), writeException); if (!mergeClusteredOperations.isEmpty()) { - dbsnpClusteredVariantOperationWriter.write(mergeClusteredOperations); + dbsnpClusteredVariantOperationWriter.write(new Chunk<>(mergeClusteredOperations)); } } } @@ -86,8 +85,8 @@ private DbsnpClusteredVariantOperationEntity buildClusteredMergeOperation(DbsnpC DbsnpClusteredVariantOperationEntity operation = new DbsnpClusteredVariantOperationEntity(); operation.fill(EventType.MERGED, origin.getAccession(), mergedInto.getAccession(), - "Identical clustered variant received multiple RS identifiers", - Arrays.asList(inactiveEntity)); + "Identical clustered variant received multiple RS identifiers", + Arrays.asList(inactiveEntity)); return operation; } diff --git a/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/batch/listeners/ImportDbsnpJsonVariantsStepProgressListener.java b/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/batch/listeners/ImportDbsnpJsonVariantsStepProgressListener.java index d1879108b..e715d93ea 100644 --- a/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/batch/listeners/ImportDbsnpJsonVariantsStepProgressListener.java +++ b/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/batch/listeners/ImportDbsnpJsonVariantsStepProgressListener.java @@ -15,7 +15,7 @@ */ package uk.ac.ebi.eva.accession.dbsnp2.batch.listeners; -import org.codehaus.jackson.JsonNode; +import com.fasterxml.jackson.databind.JsonNode; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.batch.core.ExitStatus; @@ -63,8 +63,8 @@ public ExitStatus afterStep(StepExecution stepExecution) { String stepName = stepExecution.getStepName(); long numTotalItemsRead = stepExecution.getReadCount(); logger.info("Step {} finished: Items read = {}, rs written = {}, operations written = {}", - stepName, numTotalItemsRead, - importCounts.getClusteredVariantsWritten(), importCounts.getOperationsWritten()); + stepName, numTotalItemsRead, + importCounts.getClusteredVariantsWritten(), importCounts.getOperationsWritten()); // add import counts to execution context, so they can be retrieved later if the job is restarted ExecutionContext executionContext = stepExecution.getExecutionContext(); addImportCountsToExecutionContext(executionContext); diff --git a/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/batch/processors/ContigToGenbankReplacerProcessor.java b/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/batch/processors/ContigToGenbankReplacerProcessor.java index 419ca4a18..fa4986349 100644 --- a/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/batch/processors/ContigToGenbankReplacerProcessor.java +++ b/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/batch/processors/ContigToGenbankReplacerProcessor.java @@ -19,11 +19,10 @@ import org.slf4j.LoggerFactory; import org.springframework.batch.item.ItemProcessor; import uk.ac.ebi.ampt2d.commons.accession.hashing.SHA1HashingFunction; - -import uk.ac.ebi.eva.accession.core.model.ClusteredVariant; -import uk.ac.ebi.eva.accession.core.model.IClusteredVariant; import uk.ac.ebi.eva.accession.core.contig.ContigMapping; import uk.ac.ebi.eva.accession.core.contig.ContigSynonyms; +import uk.ac.ebi.eva.accession.core.model.ClusteredVariant; +import uk.ac.ebi.eva.accession.core.model.IClusteredVariant; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpClusteredVariantEntity; import uk.ac.ebi.eva.accession.core.summary.ClusteredVariantSummaryFunction; diff --git a/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/configuration/RunnerConfiguration.java b/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/configuration/RunnerConfiguration.java index da9bb9d98..23ede5cfa 100644 --- a/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/configuration/RunnerConfiguration.java +++ b/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/configuration/RunnerConfiguration.java @@ -16,16 +16,10 @@ */ package uk.ac.ebi.eva.accession.dbsnp2.configuration; -import org.springframework.batch.core.configuration.annotation.BatchConfigurer; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; - -import uk.ac.ebi.eva.commons.batch.configuration.SpringBoot1CompatibilityConfiguration; import uk.ac.ebi.eva.commons.batch.job.JobExecutionApplicationListener; -import javax.persistence.EntityManagerFactory; -import javax.sql.DataSource; - @Configuration public class RunnerConfiguration { @@ -33,11 +27,4 @@ public class RunnerConfiguration { public JobExecutionApplicationListener jobExecutionApplicationListener() { return new JobExecutionApplicationListener(); } - - @Bean - public BatchConfigurer configurer(DataSource dataSource, EntityManagerFactory entityManagerFactory) - throws Exception { - return SpringBoot1CompatibilityConfiguration.getSpringBoot1CompatibleBatchConfigurer(dataSource, - entityManagerFactory); - } } diff --git a/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/configuration/batch/io/ImportDbsnpJsonVariantsWriterConfiguration.java b/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/configuration/batch/io/ImportDbsnpJsonVariantsWriterConfiguration.java index 4f78fc659..110c7efba 100644 --- a/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/configuration/batch/io/ImportDbsnpJsonVariantsWriterConfiguration.java +++ b/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/configuration/batch/io/ImportDbsnpJsonVariantsWriterConfiguration.java @@ -24,12 +24,12 @@ import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Import; import org.springframework.data.mongodb.core.MongoTemplate; -import uk.ac.ebi.eva.accession.core.configuration.nonhuman.MongoConfiguration; import uk.ac.ebi.eva.accession.core.batch.io.DbsnpClusteredVariantOperationWriter; import uk.ac.ebi.eva.accession.core.batch.io.DbsnpClusteredVariantWriter; import uk.ac.ebi.eva.accession.core.batch.listeners.ImportCounts; -import uk.ac.ebi.eva.accession.core.repository.nonhuman.dbsnp.DbsnpClusteredVariantAccessioningRepository; +import uk.ac.ebi.eva.accession.core.configuration.nonhuman.MongoConfiguration; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpClusteredVariantEntity; +import uk.ac.ebi.eva.accession.core.repository.nonhuman.dbsnp.DbsnpClusteredVariantAccessioningRepository; import uk.ac.ebi.eva.accession.core.repository.nonhuman.dbsnp.DbsnpClusteredVariantOperationRepository; import uk.ac.ebi.eva.accession.dbsnp2.batch.io.DbsnpJsonClusteredVariantsWriter; import uk.ac.ebi.eva.accession.dbsnp2.parameters.InputParameters; diff --git a/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/configuration/batch/jobs/ImportDbsnpJsonVariantsJobConfiguration.java b/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/configuration/batch/jobs/ImportDbsnpJsonVariantsJobConfiguration.java index 000dca4aa..d747cfa86 100644 --- a/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/configuration/batch/jobs/ImportDbsnpJsonVariantsJobConfiguration.java +++ b/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/configuration/batch/jobs/ImportDbsnpJsonVariantsJobConfiguration.java @@ -17,9 +17,10 @@ import org.springframework.batch.core.Job; import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; -import org.springframework.batch.core.configuration.annotation.JobBuilderFactory; +import org.springframework.batch.core.job.builder.JobBuilder; import org.springframework.batch.core.job.flow.Flow; import org.springframework.batch.core.launch.support.RunIdIncrementer; +import org.springframework.batch.core.repository.JobRepository; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; @@ -40,8 +41,8 @@ public class ImportDbsnpJsonVariantsJobConfiguration { private Flow importFlow; @Bean(IMPORT_DBSNP_JSON_VARIANTS_JOB) - public Job importDbsnpJsonVariantsJob(JobBuilderFactory jobBuilderFactory) { - return jobBuilderFactory.get(IMPORT_DBSNP_JSON_VARIANTS_JOB) + public Job importDbsnpJsonVariantsJob(JobRepository jobRepository) { + return new JobBuilder(IMPORT_DBSNP_JSON_VARIANTS_JOB, jobRepository) .incrementer(new RunIdIncrementer()) .start(importFlow) .end() diff --git a/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/configuration/batch/steps/ImportDbsnpJsonVariantsStepConfiguration.java b/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/configuration/batch/steps/ImportDbsnpJsonVariantsStepConfiguration.java index d53f3e607..0828b2229 100644 --- a/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/configuration/batch/steps/ImportDbsnpJsonVariantsStepConfiguration.java +++ b/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/configuration/batch/steps/ImportDbsnpJsonVariantsStepConfiguration.java @@ -19,7 +19,8 @@ import org.springframework.batch.core.Step; import org.springframework.batch.core.StepExecutionListener; import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; -import org.springframework.batch.core.configuration.annotation.StepBuilderFactory; +import org.springframework.batch.core.repository.JobRepository; +import org.springframework.batch.core.step.builder.StepBuilder; import org.springframework.batch.item.ItemProcessor; import org.springframework.batch.item.ItemWriter; import org.springframework.batch.item.file.FlatFileItemReader; @@ -28,13 +29,14 @@ import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.transaction.PlatformTransactionManager; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpClusteredVariantEntity; -import static uk.ac.ebi.eva.accession.dbsnp2.configuration.BeanNames.DBSNP_JSON_VARIANT_READER; import static uk.ac.ebi.eva.accession.dbsnp2.configuration.BeanNames.DBSNP_JSON_VARIANT_PROCESSOR; +import static uk.ac.ebi.eva.accession.dbsnp2.configuration.BeanNames.DBSNP_JSON_VARIANT_READER; import static uk.ac.ebi.eva.accession.dbsnp2.configuration.BeanNames.DBSNP_JSON_VARIANT_WRITER; -import static uk.ac.ebi.eva.accession.dbsnp2.configuration.BeanNames.IMPORT_DBSNP_JSON_VARIANTS_STEP; import static uk.ac.ebi.eva.accession.dbsnp2.configuration.BeanNames.IMPORT_DBSNP_JSON_VARIANTS_PROGRESS_LISTENER; +import static uk.ac.ebi.eva.accession.dbsnp2.configuration.BeanNames.IMPORT_DBSNP_JSON_VARIANTS_STEP; /** * Configuration for dbSNP JSON import flow step @@ -61,14 +63,14 @@ public class ImportDbsnpJsonVariantsStepConfiguration { @Bean(IMPORT_DBSNP_JSON_VARIANTS_STEP) - public Step importDbsnpJsonVariantsStep(StepBuilderFactory stepBuilderFactory, + public Step importDbsnpJsonVariantsStep(JobRepository jobRepository, PlatformTransactionManager transactionManager, SimpleCompletionPolicy chunkSizeCompletionPolicy) { - return stepBuilderFactory.get(IMPORT_DBSNP_JSON_VARIANTS_STEP) - .chunk(chunkSizeCompletionPolicy) - .reader(variantReader) - .processor(variantProcessor) - .writer(variantWriter) - .listener(importDbsnpJsonVariantsProgressListener) - .build(); + return new StepBuilder(IMPORT_DBSNP_JSON_VARIANTS_STEP, jobRepository) + .chunk(chunkSizeCompletionPolicy, transactionManager) + .reader(variantReader) + .processor(variantProcessor) + .writer(variantWriter) + .listener(importDbsnpJsonVariantsProgressListener) + .build(); } } diff --git a/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/runner/DbsnpJsonImportVariantsJobLauncherCommandLineRunner.java b/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/runner/DbsnpJsonImportVariantsJobLauncherCommandLineRunner.java index 198f4246a..ff80ea812 100644 --- a/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/runner/DbsnpJsonImportVariantsJobLauncherCommandLineRunner.java +++ b/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/runner/DbsnpJsonImportVariantsJobLauncherCommandLineRunner.java @@ -24,7 +24,6 @@ import org.springframework.batch.core.JobParametersInvalidException; import org.springframework.batch.core.explore.JobExplorer; import org.springframework.batch.core.launch.JobLauncher; -import org.springframework.batch.core.launch.JobParametersNotFoundException; import org.springframework.batch.core.repository.JobExecutionAlreadyRunningException; import org.springframework.batch.core.repository.JobInstanceAlreadyCompleteException; import org.springframework.batch.core.repository.JobRepository; @@ -32,11 +31,10 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.ExitCodeGenerator; -import org.springframework.boot.autoconfigure.batch.JobLauncherCommandLineRunner; +import org.springframework.boot.autoconfigure.batch.JobLauncherApplicationRunner; import org.springframework.context.ApplicationEventPublisherAware; import org.springframework.stereotype.Component; import org.springframework.util.PatternMatchUtils; - import uk.ac.ebi.eva.accession.core.runner.CommandLineRunnerUtils; import uk.ac.ebi.eva.accession.dbsnp2.parameters.InputParameters; import uk.ac.ebi.eva.commons.batch.exception.NoJobToExecuteException; @@ -51,7 +49,7 @@ * Custom job launcher command line runner to integrate Job with Input Parameters */ @Component -public class DbsnpJsonImportVariantsJobLauncherCommandLineRunner extends JobLauncherCommandLineRunner implements +public class DbsnpJsonImportVariantsJobLauncherCommandLineRunner extends JobLauncherApplicationRunner implements ApplicationEventPublisherAware, ExitCodeGenerator { private static final Logger logger = LoggerFactory @@ -81,7 +79,7 @@ public class DbsnpJsonImportVariantsJobLauncherCommandLineRunner extends JobLaun private boolean abnormalExit; public DbsnpJsonImportVariantsJobLauncherCommandLineRunner(JobLauncher jobLauncher, JobExplorer jobExplorer, - JobRepository jobRepository) { + JobRepository jobRepository) { super(jobLauncher, jobExplorer, jobRepository); this.jobExplorer = jobExplorer; this.jobRepository = jobRepository; @@ -94,9 +92,9 @@ public void setJobs(Collection jobs) { } @Override - public void setJobNames(String jobName) { + public void setJobName(String jobName) { this.jobName = jobName; - super.setJobNames(jobName); + super.setJobName(jobName); } @Override @@ -119,16 +117,15 @@ public void run(String... args) throws JobExecutionException { JobStatusManager.checkIfPropertiesHaveBeenProvided(jobParameters); if (inputParameters.isForceRestart()) { JobExecution previousJobExecution = CommandLineRunnerUtils.getLastJobExecution(jobName, jobExplorer, - jobParameters); + jobParameters); CommandLineRunnerUtils.markPreviousJobAsFailed(jobName, jobRepository, previousJobExecution != null ? previousJobExecution.getJobParameters() : jobParameters); - } - else { + } else { jobParameters = CommandLineRunnerUtils.addRunIDToJobParameters(jobName, jobExplorer, jobParameters); } launchJob(jobParameters); } catch (NoPreviousJobExecutionException | NoParametersHaveBeenPassedException | NoJobToExecuteException - | JobParametersInvalidException | JobExecutionAlreadyRunningException e) { + | JobParametersInvalidException | JobExecutionAlreadyRunningException e) { logger.error(e.getMessage()); logger.debug("Error trace", e); abnormalExit = true; @@ -147,8 +144,7 @@ private void launchJob(JobParameters jobParameters) throws JobExecutionException @Override protected void execute(Job job, JobParameters jobParameters) throws JobExecutionAlreadyRunningException, - JobRestartException, JobInstanceAlreadyCompleteException, JobParametersInvalidException, - JobParametersNotFoundException { + JobRestartException, JobInstanceAlreadyCompleteException, JobParametersInvalidException { logger.info("Running job '" + jobName + "' with parameters: " + jobParameters); super.execute(job, jobParameters); } diff --git a/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/batch/io/DbsnpJsonClusteredVariantsWriterTest.java b/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/batch/io/DbsnpJsonClusteredVariantsWriterTest.java index 399fd2382..8c2b413dd 100644 --- a/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/batch/io/DbsnpJsonClusteredVariantsWriterTest.java +++ b/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/batch/io/DbsnpJsonClusteredVariantsWriterTest.java @@ -15,9 +15,10 @@ */ package uk.ac.ebi.eva.accession.dbsnp2.batch.io; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.batch.item.Chunk; import org.springframework.batch.test.StepScopeTestExecutionListener; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.mongodb.core.MongoTemplate; @@ -25,22 +26,24 @@ import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestExecutionListeners; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import org.springframework.test.context.support.DependencyInjectionTestExecutionListener; import uk.ac.ebi.ampt2d.commons.accession.core.models.EventType; import uk.ac.ebi.ampt2d.commons.accession.hashing.SHA1HashingFunction; -import uk.ac.ebi.eva.accession.core.model.ClusteredVariant; -import uk.ac.ebi.eva.accession.core.model.IClusteredVariant; import uk.ac.ebi.eva.accession.core.batch.io.DbsnpClusteredVariantOperationWriter; import uk.ac.ebi.eva.accession.core.batch.io.DbsnpClusteredVariantWriter; import uk.ac.ebi.eva.accession.core.batch.listeners.ImportCounts; -import uk.ac.ebi.eva.accession.core.repository.nonhuman.dbsnp.DbsnpClusteredVariantAccessioningRepository; +import uk.ac.ebi.eva.accession.core.model.ClusteredVariant; +import uk.ac.ebi.eva.accession.core.model.IClusteredVariant; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpClusteredVariantEntity; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpClusteredVariantInactiveEntity; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpClusteredVariantOperationEntity; +import uk.ac.ebi.eva.accession.core.repository.nonhuman.dbsnp.DbsnpClusteredVariantAccessioningRepository; import uk.ac.ebi.eva.accession.core.repository.nonhuman.dbsnp.DbsnpClusteredVariantOperationRepository; import uk.ac.ebi.eva.accession.core.summary.ClusteredVariantSummaryFunction; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.dbsnp2.test.BatchTestConfiguration; +import uk.ac.ebi.eva.accession.dbsnp2.test.MongoTestConfiguration; import uk.ac.ebi.eva.commons.core.models.VariantType; import java.time.LocalDateTime; @@ -49,15 +52,15 @@ import java.util.List; import java.util.function.Function; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; -@RunWith(SpringRunner.class) -@ContextConfiguration(classes = {BatchTestConfiguration.class}) +@ExtendWith(SpringExtension.class) +@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class}) @TestExecutionListeners({DependencyInjectionTestExecutionListener.class, StepScopeTestExecutionListener.class}) @TestPropertySource({"classpath:application.properties"}) -public class DbsnpJsonClusteredVariantsWriterTest { +public class DbsnpJsonClusteredVariantsWriterTest extends MongoTestContainerHelper { private DbsnpJsonClusteredVariantsWriter dbsnpJsonClusteredVariantsWriter; @@ -76,91 +79,91 @@ public class DbsnpJsonClusteredVariantsWriterTest { private DbsnpClusteredVariantEntity variantEntity2; private Function hashingFuncClustered = - new ClusteredVariantSummaryFunction().andThen(new SHA1HashingFunction()); + new ClusteredVariantSummaryFunction().andThen(new SHA1HashingFunction()); - @Before + @BeforeEach public void setUp() { importCounts = new ImportCounts(); DbsnpClusteredVariantWriter dbsnpClusteredVariantWriter = new DbsnpClusteredVariantWriter(mongoTemplate, - importCounts); + importCounts); DbsnpClusteredVariantOperationWriter dbsnpClusteredVariantOperationWriter = new DbsnpClusteredVariantOperationWriter(mongoTemplate, importCounts); variantEntity1 = buildClusteredVariantEntity(1L, - buildClusteredVariant("acsn1", - "contig1", - 1L, - VariantType.SNV)); + buildClusteredVariant("acsn1", + "contig1", + 1L, + VariantType.SNV)); dbsnpJsonClusteredVariantsWriter = new DbsnpJsonClusteredVariantsWriter(dbsnpClusteredVariantWriter, dbsnpClusteredVariantOperationWriter, - dbsnpClusteredVariantOperationRepository, - dbsnpClusteredVariantAccessioningRepository); + dbsnpClusteredVariantOperationRepository, + dbsnpClusteredVariantAccessioningRepository); mongoTemplate.dropCollection(DbsnpClusteredVariantEntity.class); mongoTemplate.dropCollection(DbsnpClusteredVariantOperationEntity.class); } @Test public void writeSingleVariant() throws Exception { - dbsnpJsonClusteredVariantsWriter.write(Collections.singletonList(variantEntity1)); + dbsnpJsonClusteredVariantsWriter.write(Chunk.of(variantEntity1)); assertClusteredVariantStored(1, Collections.singletonList(variantEntity1)); } @Test public void writeMultipleVariantsWithDifferentContig() throws Exception { variantEntity2 = buildClusteredVariantEntity(variantEntity1.getAccession(), - buildClusteredVariant(variantEntity1.getAssemblyAccession(), - "contig2", - variantEntity1.getStart(), - variantEntity1.getType())); + buildClusteredVariant(variantEntity1.getAssemblyAccession(), + "contig2", + variantEntity1.getStart(), + variantEntity1.getType())); List clusteredVariantEntities = Arrays.asList(variantEntity1, variantEntity2); - dbsnpJsonClusteredVariantsWriter.write(clusteredVariantEntities); + dbsnpJsonClusteredVariantsWriter.write(new Chunk<>(clusteredVariantEntities)); assertClusteredVariantStored(2, clusteredVariantEntities); } @Test public void writeMultipleVariantsWithDifferentStart() throws Exception { variantEntity2 = buildClusteredVariantEntity(variantEntity1.getAccession(), - buildClusteredVariant(variantEntity1.getAssemblyAccession(), - variantEntity1.getContig(), - 2L, - variantEntity1.getType())); + buildClusteredVariant(variantEntity1.getAssemblyAccession(), + variantEntity1.getContig(), + 2L, + variantEntity1.getType())); List clusteredVariantEntities = Arrays.asList(variantEntity1, variantEntity2); - dbsnpJsonClusteredVariantsWriter.write(clusteredVariantEntities); + dbsnpJsonClusteredVariantsWriter.write(new Chunk<>(clusteredVariantEntities)); assertClusteredVariantStored(2, clusteredVariantEntities); } @Test public void writeMultipleVariantsWithDifferentVariantType() throws Exception { variantEntity2 = buildClusteredVariantEntity(variantEntity1.getAccession(), - buildClusteredVariant(variantEntity1.getAssemblyAccession(), - variantEntity1.getContig(), - variantEntity1.getStart(), - VariantType.DEL)); + buildClusteredVariant(variantEntity1.getAssemblyAccession(), + variantEntity1.getContig(), + variantEntity1.getStart(), + VariantType.DEL)); List clusteredVariantEntities = Arrays.asList(variantEntity1, variantEntity2); - dbsnpJsonClusteredVariantsWriter.write(clusteredVariantEntities); + dbsnpJsonClusteredVariantsWriter.write(new Chunk<>(clusteredVariantEntities)); assertClusteredVariantStored(2, clusteredVariantEntities); } @Test public void writeMultipleVariantsWithDifferentAssemblyAccession() throws Exception { variantEntity2 = buildClusteredVariantEntity(variantEntity1.getAccession(), - buildClusteredVariant("acsn2", - variantEntity1.getContig(), - variantEntity1.getStart(), - variantEntity1.getType())); + buildClusteredVariant("acsn2", + variantEntity1.getContig(), + variantEntity1.getStart(), + variantEntity1.getType())); List clusteredVariantEntities = Arrays.asList(variantEntity1, variantEntity2); - dbsnpJsonClusteredVariantsWriter.write(clusteredVariantEntities); + dbsnpJsonClusteredVariantsWriter.write(new Chunk<>(clusteredVariantEntities)); assertClusteredVariantStored(2, clusteredVariantEntities); } @Test public void writeMultipleVariantsWithDifferentRsIDWithSameClusteredVariant() throws Exception { variantEntity2 = buildClusteredVariantEntity(2L, - variantEntity1.getModel()); + variantEntity1.getModel()); DbsnpClusteredVariantEntity variantEntity3 = buildClusteredVariantEntity(3L, - variantEntity1.getModel()); + variantEntity1.getModel()); List clusteredVariantEntities = Arrays.asList(variantEntity1, variantEntity2, - variantEntity3); - dbsnpJsonClusteredVariantsWriter.write(clusteredVariantEntities); + variantEntity3); + dbsnpJsonClusteredVariantsWriter.write(new Chunk<>(clusteredVariantEntities)); assertClusteredVariantStored(1, clusteredVariantEntities); DbsnpClusteredVariantOperationEntity clusteredVariantOperationEntity1 = @@ -168,21 +171,21 @@ public void writeMultipleVariantsWithDifferentRsIDWithSameClusteredVariant() thr DbsnpClusteredVariantOperationEntity clusteredVariantOperationEntity2 = new DbsnpClusteredVariantOperationEntity(); clusteredVariantOperationEntity1.fill(EventType.MERGED, variantEntity2.getAccession(), - variantEntity1.getAccession(), - "Identical clustered variant received multiple RS identifiers", - Arrays.asList(new DbsnpClusteredVariantInactiveEntity(variantEntity2))); + variantEntity1.getAccession(), + "Identical clustered variant received multiple RS identifiers", + Arrays.asList(new DbsnpClusteredVariantInactiveEntity(variantEntity2))); clusteredVariantOperationEntity2.fill(EventType.MERGED, variantEntity3.getAccession(), - variantEntity1.getAccession(), - "Identical clustered variant received multiple RS identifiers", - Arrays.asList(new DbsnpClusteredVariantInactiveEntity(variantEntity3))); + variantEntity1.getAccession(), + "Identical clustered variant received multiple RS identifiers", + Arrays.asList(new DbsnpClusteredVariantInactiveEntity(variantEntity3))); assertClusteredVariantOperationStored(2, Arrays.asList(clusteredVariantOperationEntity1, - clusteredVariantOperationEntity2)); + clusteredVariantOperationEntity2)); } @Test public void writeDuplicateVariantShouldNotBeStored() throws Exception { List clusteredVariantEntities = Arrays.asList(variantEntity1, variantEntity1); - dbsnpJsonClusteredVariantsWriter.write(clusteredVariantEntities); + dbsnpJsonClusteredVariantsWriter.write(new Chunk<>(clusteredVariantEntities)); assertClusteredVariantStored(1, clusteredVariantEntities); } @@ -193,22 +196,22 @@ private DbsnpClusteredVariantEntity buildClusteredVariantEntity(Long accession, private IClusteredVariant buildClusteredVariant(String accession, String contig, long start, VariantType variantType) { return new ClusteredVariant(accession, - 9606, - contig, - start, - variantType, - Boolean.FALSE, - LocalDateTime.now()); + 9606, + contig, + start, + variantType, + Boolean.FALSE, + LocalDateTime.now()); } private void assertClusteredVariantStored(int expectedVariants, List clusteredVariantEntities) { List actualClusteredVariantEntities = mongoTemplate.find - (new Query(), DbsnpClusteredVariantEntity.class); + (new Query(), DbsnpClusteredVariantEntity.class); assertEquals(expectedVariants, actualClusteredVariantEntities.size()); assertEquals(expectedVariants, importCounts.getClusteredVariantsWritten()); clusteredVariantEntities.forEach(entity -> - assertTrue(actualClusteredVariantEntities.contains(entity))); + assertTrue(actualClusteredVariantEntities.contains(entity))); } private void assertClusteredVariantOperationStored @@ -223,7 +226,7 @@ private void assertClusteredVariantStored(int expectedVariants, } private boolean dbsnpClusteredVariantOperationEntitiesEqual(DbsnpClusteredVariantOperationEntity entity1, - DbsnpClusteredVariantOperationEntity entity2) { + DbsnpClusteredVariantOperationEntity entity2) { return entity1.getAccession().equals(entity2.getAccession()) && entity1.getMergedInto().equals(entity2.getMergedInto()) && entity1.getInactiveObjects().containsAll(entity2.getInactiveObjects()) && diff --git a/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/batch/io/DbsnpJsonItemReaderTest.java b/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/batch/io/DbsnpJsonItemReaderTest.java index 3e6f60b98..bd95e7e1d 100644 --- a/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/batch/io/DbsnpJsonItemReaderTest.java +++ b/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/batch/io/DbsnpJsonItemReaderTest.java @@ -16,10 +16,8 @@ package uk.ac.ebi.eva.accession.dbsnp2.batch.io; import com.fasterxml.jackson.databind.JsonNode; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.batch.item.ExecutionContext; import org.springframework.batch.item.ItemStreamException; import org.springframework.batch.item.file.FlatFileItemReader; @@ -29,7 +27,7 @@ import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestExecutionListeners; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import org.springframework.test.context.support.DependencyInjectionTestExecutionListener; import uk.ac.ebi.eva.accession.dbsnp2.test.BatchTestConfiguration; @@ -37,11 +35,12 @@ import java.util.ArrayList; import java.util.List; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; import static uk.ac.ebi.eva.accession.dbsnp2.configuration.BeanNames.DBSNP_JSON_VARIANT_READER; -@RunWith(SpringRunner.class) +@ExtendWith(SpringExtension.class) @ContextConfiguration(classes = {BatchTestConfiguration.class}) @TestExecutionListeners({DependencyInjectionTestExecutionListener.class, StepScopeTestExecutionListener.class}) @@ -52,13 +51,10 @@ public class DbsnpJsonItemReaderTest { @Qualifier(DBSNP_JSON_VARIANT_READER) private FlatFileItemReader reader; - @Rule - public ExpectedException thrown = ExpectedException.none(); - @Test public void readExistingSource() throws Exception { reader.setResource(new BzipLazyResource( - new File("src/test/resources/input-files/test-dbsnp.json.bz2"))); + new File("src/test/resources/input-files/test-dbsnp.json.bz2"))); reader.open(new ExecutionContext()); List variants = readAll(reader); assertEquals(26, variants.size()); @@ -67,8 +63,7 @@ public void readExistingSource() throws Exception { @Test public void readNonExistingSource() throws Exception { reader.setResource(new BzipLazyResource(new File("INVALID_DIRECTORY"))); - thrown.expect(ItemStreamException.class); - reader.open(new ExecutionContext()); + assertThrows(ItemStreamException.class, () -> reader.open(new ExecutionContext())); } private List readAll(FlatFileItemReader reader) throws Exception { diff --git a/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/batch/processors/ContigToGenbankReplacerProcessorTest.java b/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/batch/processors/ContigToGenbankReplacerProcessorTest.java index 8de028099..253688b54 100644 --- a/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/batch/processors/ContigToGenbankReplacerProcessorTest.java +++ b/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/batch/processors/ContigToGenbankReplacerProcessorTest.java @@ -15,15 +15,12 @@ */ package uk.ac.ebi.eva.accession.dbsnp2.batch.processors; -import org.junit.Before; -import org.junit.Rule; -import org.junit.rules.ExpectedException; -import org.junit.Test; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import uk.ac.ebi.ampt2d.commons.accession.hashing.SHA1HashingFunction; - +import uk.ac.ebi.eva.accession.core.contig.ContigMapping; import uk.ac.ebi.eva.accession.core.model.ClusteredVariant; import uk.ac.ebi.eva.accession.core.model.IClusteredVariant; -import uk.ac.ebi.eva.accession.core.contig.ContigMapping; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpClusteredVariantEntity; import uk.ac.ebi.eva.accession.core.summary.ClusteredVariantSummaryFunction; import uk.ac.ebi.eva.commons.core.models.VariantType; @@ -31,7 +28,8 @@ import java.time.LocalDateTime; import java.util.function.Function; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; public class ContigToGenbankReplacerProcessorTest { @@ -39,11 +37,8 @@ public class ContigToGenbankReplacerProcessorTest { private Function hashingFunction = new ClusteredVariantSummaryFunction().andThen(new SHA1HashingFunction()); - - @Rule - public ExpectedException thrown = ExpectedException.none(); - @Before + @BeforeEach public void setUp() throws Exception { String fileString = ContigToGenbankReplacerProcessorTest.class.getResource( "/input-files/GCF_000001405.38_GRCh38.p12_assembly_report.txt").toString(); @@ -53,19 +48,19 @@ public void setUp() throws Exception { } @Test - public void contigGenbank() throws Exception { + public void contigGenbank() { DbsnpClusteredVariantEntity variant = buildMockVariant("CM000686.2"); assertEquals("CM000686.2", processor.process(variant).getContig()); } @Test - public void contigChrToGenbank() throws Exception { + public void contigChrToGenbank() { DbsnpClusteredVariantEntity variant = buildMockVariant("chrY"); assertEquals("CM000686.2", processor.process(variant).getContig()); } @Test - public void contigRefseqToGenbank() throws Exception { + public void contigRefseqToGenbank() { DbsnpClusteredVariantEntity variant = buildMockVariant("NC_000024.10"); assertEquals("CM000686.2", processor.process(variant).getContig()); } @@ -73,71 +68,69 @@ public void contigRefseqToGenbank() throws Exception { @Test public void genbankAndRefseqNotEquivalents() throws Exception { DbsnpClusteredVariantEntity variant = buildMockVariant("chr3"); - thrown.expect(IllegalStateException.class); - assertEquals("chr3", processor.process(variant).getContig()); + assertThrows(IllegalStateException.class, () -> processor.process(variant)); } @Test - public void genbankAndRefseqNotEquivalentsRefseqNotPresent() throws Exception { + public void genbankAndRefseqNotEquivalentsRefseqNotPresent() { DbsnpClusteredVariantEntity variant = buildMockVariant("chrUn_KI270752v1"); assertEquals("KI270752.1", processor.process(variant).getContig()); } @Test - public void genbankAndRefseqNotEquivalentsGenbankNotPresent() throws Exception { + public void genbankAndRefseqNotEquivalentsGenbankNotPresent() { DbsnpClusteredVariantEntity variant = buildMockVariant("tstchr2"); assertEquals("tstchr2", processor.process(variant).getContig()); } @Test - public void genbankAndRefseqNotEquivalentsNonePresent() throws Exception { + public void genbankAndRefseqNotEquivalentsNonePresent() { DbsnpClusteredVariantEntity variant = buildMockVariant("tstchr3"); assertEquals("tstchr3", processor.process(variant).getContig()); } @Test - public void contigNotFoundInAssemblyReport() throws Exception { + public void contigNotFoundInAssemblyReport() { DbsnpClusteredVariantEntity variant = buildMockVariant("chr"); - thrown.expect(IllegalStateException.class); - assertEquals("chr", processor.process(variant).getContig()); + assertThrows(IllegalStateException.class, () -> processor.process(variant)); } @Test - public void noGenbankDontConvert() throws Exception { + public void noGenbankDontConvert() { DbsnpClusteredVariantEntity variant = buildMockVariant("tstchr4"); assertEquals("tstchr4", processor.process(variant).getContig()); } @Test - public void updatedHashedMessageAfterContigReplacement() throws Exception { + public void updatedHashedMessageAfterContigReplacement() { DbsnpClusteredVariantEntity processedVariant = processor.process(buildMockVariant("NC_000024.10")); assertEquals(new SHA1HashingFunction().apply("1_CM000686.2_1_SNV"), - processedVariant.getHashedMessage()); + processedVariant.getHashedMessage()); processedVariant = processor.process(buildMockVariant("chrY")); assertEquals(new SHA1HashingFunction().apply("1_CM000686.2_1_SNV"), - processedVariant.getHashedMessage()); + processedVariant.getHashedMessage()); processedVariant = processor.process(buildMockVariant("chrUn_KI270752v1")); assertEquals(new SHA1HashingFunction().apply("1_KI270752.1_1_SNV"), - processedVariant.getHashedMessage()); + processedVariant.getHashedMessage()); processedVariant = processor.process(buildMockVariant("tstchr2")); assertEquals(new SHA1HashingFunction().apply("1_tstchr2_1_SNV"), - processedVariant.getHashedMessage()); + processedVariant.getHashedMessage()); } private DbsnpClusteredVariantEntity buildMockVariant(String originalChromosome) { ClusteredVariant newVariant = new ClusteredVariant("1", - 9606, - originalChromosome, - 1, - VariantType.SNV, - Boolean.FALSE, - LocalDateTime.now()); + 9606, + originalChromosome, + 1, + VariantType.SNV, + Boolean.FALSE, + LocalDateTime.now()); return new DbsnpClusteredVariantEntity(1L, - hashingFunction.apply(newVariant), - newVariant, - 1); + hashingFunction.apply(newVariant), + newVariant, + 1); } } diff --git a/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/batch/processors/JsonNodeToClusteredVariantProcessorTest.java b/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/batch/processors/JsonNodeToClusteredVariantProcessorTest.java index 2cb816149..0b8e3f97a 100644 --- a/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/batch/processors/JsonNodeToClusteredVariantProcessorTest.java +++ b/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/batch/processors/JsonNodeToClusteredVariantProcessorTest.java @@ -16,11 +16,9 @@ package uk.ac.ebi.eva.accession.dbsnp2.batch.processors; import com.fasterxml.jackson.databind.JsonNode; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.batch.item.ExecutionContext; import org.springframework.batch.item.file.FlatFileItemReader; import org.springframework.batch.test.StepScopeTestExecutionListener; @@ -29,12 +27,11 @@ import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestExecutionListeners; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import org.springframework.test.context.support.DependencyInjectionTestExecutionListener; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpClusteredVariantEntity; import uk.ac.ebi.eva.accession.dbsnp2.batch.io.BzipLazyResource; import uk.ac.ebi.eva.accession.dbsnp2.batch.io.JsonNodeLineMapper; -import uk.ac.ebi.eva.accession.dbsnp2.batch.processors.JsonNodeToClusteredVariantProcessor; import uk.ac.ebi.eva.accession.dbsnp2.parameters.InputParameters; import uk.ac.ebi.eva.accession.dbsnp2.test.BatchTestConfiguration; import uk.ac.ebi.eva.commons.core.models.VariantType; @@ -45,11 +42,11 @@ import java.util.Objects; import java.util.stream.Collectors; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; import static uk.ac.ebi.eva.accession.dbsnp2.configuration.BeanNames.DBSNP_JSON_VARIANT_READER; -@RunWith(SpringRunner.class) +@ExtendWith(SpringExtension.class) @ContextConfiguration(classes = {BatchTestConfiguration.class}) @TestExecutionListeners({DependencyInjectionTestExecutionListener.class, StepScopeTestExecutionListener.class}) @@ -63,10 +60,8 @@ public class JsonNodeToClusteredVariantProcessorTest { private InputParameters inputParameters; private JsonNodeToClusteredVariantProcessor processor; private List variants = new ArrayList<>(); - @Rule - public ExpectedException thrown = ExpectedException.none(); - @Before + @BeforeEach public void setUp() throws Exception { reader.setResource(new BzipLazyResource( new File("src/test/resources/input-files/test-dbsnp.json.bz2"))); diff --git a/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/configuration/jobs/ImportDbsnpJsonVariantsJobConfigurationTest.java b/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/configuration/jobs/ImportDbsnpJsonVariantsJobConfigurationTest.java index 64f02cbde..eb4fcded3 100644 --- a/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/configuration/jobs/ImportDbsnpJsonVariantsJobConfigurationTest.java +++ b/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/configuration/jobs/ImportDbsnpJsonVariantsJobConfigurationTest.java @@ -15,37 +15,44 @@ */ package uk.ac.ebi.eva.accession.dbsnp2.configuration.jobs; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.batch.core.BatchStatus; import org.springframework.batch.core.JobExecution; import org.springframework.batch.core.StepExecution; import org.springframework.batch.test.JobLauncherTestUtils; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; - +import org.springframework.test.context.junit.jupiter.SpringExtension; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.dbsnp2.parameters.InputParameters; +import uk.ac.ebi.eva.accession.dbsnp2.test.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.accession.dbsnp2.test.BatchTestConfiguration; +import uk.ac.ebi.eva.accession.dbsnp2.test.MongoTestConfiguration; import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.stream.Collectors; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import static uk.ac.ebi.eva.accession.dbsnp2.configuration.BeanNames.IMPORT_DBSNP_JSON_VARIANTS_STEP; +import static uk.ac.ebi.eva.accession.dbsnp2.test.BatchTestConfiguration.JOB_IMPORT_DBSNP_JSON_VARIANTS_JOB; -@RunWith(SpringRunner.class) -@ContextConfiguration(classes = {BatchTestConfiguration.class}) +@ExtendWith(SpringExtension.class) +@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class, + BatchJobRepositoryTestConfiguration.class}) @TestPropertySource("classpath:application.properties") -public class ImportDbsnpJsonVariantsJobConfigurationTest { +public class ImportDbsnpJsonVariantsJobConfigurationTest extends MongoTestContainerHelper { @Autowired private InputParameters inputParameters; + @Autowired + @Qualifier(JOB_IMPORT_DBSNP_JSON_VARIANTS_JOB) private JobLauncherTestUtils jobLauncherTestUtils; @Test diff --git a/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/configuration/steps/ImportDbsnpJsonVariantsStepConfigurationTest.java b/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/configuration/steps/ImportDbsnpJsonVariantsStepConfigurationTest.java index bcf488998..06522e338 100644 --- a/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/configuration/steps/ImportDbsnpJsonVariantsStepConfigurationTest.java +++ b/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/configuration/steps/ImportDbsnpJsonVariantsStepConfigurationTest.java @@ -15,36 +15,44 @@ */ package uk.ac.ebi.eva.accession.dbsnp2.configuration.steps; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.batch.core.BatchStatus; import org.springframework.batch.core.JobExecution; import org.springframework.batch.test.JobLauncherTestUtils; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.data.mongodb.core.query.Query; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpClusteredVariantEntity; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; +import uk.ac.ebi.eva.accession.dbsnp2.test.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.accession.dbsnp2.test.BatchTestConfiguration; +import uk.ac.ebi.eva.accession.dbsnp2.test.MongoTestConfiguration; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import static uk.ac.ebi.eva.accession.dbsnp2.configuration.BeanNames.IMPORT_DBSNP_JSON_VARIANTS_STEP; +import static uk.ac.ebi.eva.accession.dbsnp2.test.BatchTestConfiguration.JOB_IMPORT_DBSNP_JSON_VARIANTS_JOB; -@RunWith(SpringRunner.class) -@ContextConfiguration(classes = {BatchTestConfiguration.class}) +@ExtendWith(SpringExtension.class) +@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class, + BatchJobRepositoryTestConfiguration.class}) @TestPropertySource("classpath:application.properties") -public class ImportDbsnpJsonVariantsStepConfigurationTest { +public class ImportDbsnpJsonVariantsStepConfigurationTest extends MongoTestContainerHelper { @Autowired + @Qualifier(JOB_IMPORT_DBSNP_JSON_VARIANTS_JOB) private JobLauncherTestUtils jobLauncherTestUtils; + @Autowired private MongoTemplate mongoTemplate; - @Before + @BeforeEach public void setUp() { mongoTemplate.dropCollection(DbsnpClusteredVariantEntity.class); } diff --git a/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/runner/DbsnpJsonImportVariantsJobLauncherCommandLineRunnerTest.java b/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/runner/DbsnpJsonImportVariantsJobLauncherCommandLineRunnerTest.java index be2692ac6..3add32b31 100644 --- a/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/runner/DbsnpJsonImportVariantsJobLauncherCommandLineRunnerTest.java +++ b/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/runner/DbsnpJsonImportVariantsJobLauncherCommandLineRunnerTest.java @@ -16,12 +16,12 @@ package uk.ac.ebi.eva.accession.dbsnp2.runner; import com.fasterxml.jackson.databind.JsonNode; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.batch.core.JobInstance; import org.springframework.batch.core.StepExecution; import org.springframework.batch.core.explore.JobExplorer; @@ -33,15 +33,16 @@ import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; - +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.eva.accession.core.runner.CommandLineRunnerUtils; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.dbsnp2.batch.io.BzipLazyResource; import uk.ac.ebi.eva.accession.dbsnp2.parameters.InputParameters; +import uk.ac.ebi.eva.accession.dbsnp2.test.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.accession.dbsnp2.test.BatchTestConfiguration; +import uk.ac.ebi.eva.accession.dbsnp2.test.MongoTestConfiguration; import uk.ac.ebi.eva.commons.core.utils.FileUtils; -import javax.sql.DataSource; import java.io.BufferedReader; import java.io.File; import java.io.IOException; @@ -52,18 +53,19 @@ import java.util.Arrays; import java.util.Collections; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; import static uk.ac.ebi.eva.accession.dbsnp2.configuration.BeanNames.IMPORT_DBSNP_JSON_VARIANTS_JOB; import static uk.ac.ebi.eva.accession.dbsnp2.configuration.BeanNames.IMPORT_DBSNP_JSON_VARIANTS_STEP; import static uk.ac.ebi.eva.accession.dbsnp2.runner.DbsnpJsonImportVariantsJobLauncherCommandLineRunner.EXIT_WITHOUT_ERRORS; import static uk.ac.ebi.eva.accession.dbsnp2.runner.DbsnpJsonImportVariantsJobLauncherCommandLineRunner.EXIT_WITH_ERRORS; -@RunWith(SpringRunner.class) -@ContextConfiguration(classes = {BatchTestConfiguration.class}) +@ExtendWith(SpringExtension.class) +@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class, + BatchJobRepositoryTestConfiguration.class}) @TestPropertySource("classpath:application.properties") @SpringBatchTest -public class DbsnpJsonImportVariantsJobLauncherCommandLineRunnerTest { +public class DbsnpJsonImportVariantsJobLauncherCommandLineRunnerTest extends MongoTestContainerHelper { @Autowired private InputParameters inputParameters; @@ -74,9 +76,6 @@ public class DbsnpJsonImportVariantsJobLauncherCommandLineRunnerTest { @Autowired private JobExplorer jobExplorer; - @Autowired - private DataSource datasource; - @Autowired private DbsnpJsonImportVariantsJobLauncherCommandLineRunner runner; @@ -93,17 +92,17 @@ public class DbsnpJsonImportVariantsJobLauncherCommandLineRunnerTest { private boolean originalInputParametersCaptured = false; - @BeforeClass + @BeforeAll public static void initializeTempFile() throws Exception { tempJsonInputFileToTestFailingJobs = File.createTempFile("resumeFailingJob", ".json.bz2"); } - @AfterClass - public static void deleteTempFile() throws Exception { + @AfterAll + public static void deleteTempFile() { tempJsonInputFileToTestFailingJobs.delete(); } - @Before + @BeforeEach public void setUp() throws Exception { if (!originalInputParametersCaptured) { originalJsonInputFilePath = inputParameters.getInput(); @@ -111,14 +110,14 @@ public void setUp() throws Exception { writeToTempJsonFile(originalJsonContent); originalInputParametersCaptured = true; } - jobRepositoryTestUtils = new JobRepositoryTestUtils(jobRepository, datasource); - runner.setJobNames(IMPORT_DBSNP_JSON_VARIANTS_JOB); + jobRepositoryTestUtils = new JobRepositoryTestUtils(jobRepository); + runner.setJobName(IMPORT_DBSNP_JSON_VARIANTS_JOB); jobRepositoryTestUtils.removeJobExecutions(); inputParameters.setForceRestart(false); useOriginalJsonFile(); } - @After + @AfterEach public void tearDown() { jobRepositoryTestUtils.removeJobExecutions(); inputParameters.setForceRestart(false); @@ -159,11 +158,11 @@ private JobInstance runJobAandCheckResults() throws Exception { runner.run(); assertEquals(EXIT_WITH_ERRORS, runner.getExitCode()); JobInstance currentJobInstance = CommandLineRunnerUtils.getLastJobExecution(IMPORT_DBSNP_JSON_VARIANTS_JOB, - jobExplorer, - inputParameters.toJobParameters()) - .getJobInstance(); + jobExplorer, + inputParameters.toJobParameters()) + .getJobInstance(); StepExecution stepExecution = jobRepository.getLastStepExecution(currentJobInstance, - IMPORT_DBSNP_JSON_VARIANTS_STEP); + IMPORT_DBSNP_JSON_VARIANTS_STEP); //Ensure that only the first batch was written (batch size is 2 and error was at line#4) assertEquals(inputParameters.getChunkSize(), stepExecution.getWriteCount()); @@ -174,9 +173,9 @@ private void runJobBAndCheckRestart(JobInstance failingJobInstance) throws Excep runner.run(); assertEquals(EXIT_WITHOUT_ERRORS, runner.getExitCode()); JobInstance currentJobInstance = CommandLineRunnerUtils.getLastJobExecution(IMPORT_DBSNP_JSON_VARIANTS_JOB, - jobExplorer, - inputParameters.toJobParameters()) - .getJobInstance(); + jobExplorer, + inputParameters.toJobParameters()) + .getJobInstance(); assertNotEquals(failingJobInstance.getInstanceId(), currentJobInstance.getInstanceId()); } @@ -222,11 +221,11 @@ private void runJobBAndCheckResults() throws Exception { private void runJobCAndCheckResumption(JobInstance failingJobInstance) throws Exception { runner.run(); JobInstance currentJobInstance = CommandLineRunnerUtils.getLastJobExecution(IMPORT_DBSNP_JSON_VARIANTS_JOB, - jobExplorer, - inputParameters.toJobParameters()) - .getJobInstance(); + jobExplorer, + inputParameters.toJobParameters()) + .getJobInstance(); StepExecution stepExecution = jobRepository.getLastStepExecution(currentJobInstance, - IMPORT_DBSNP_JSON_VARIANTS_STEP); + IMPORT_DBSNP_JSON_VARIANTS_STEP); // Did we resume the previous failed job instance? assertEquals(failingJobInstance.getInstanceId(), currentJobInstance.getInstanceId()); @@ -294,7 +293,7 @@ private String getOriginalJsonContent(String inputJsonPath) throws IOException { private int getNumberOfLinesInJsonString(String jsonString) { return (int) Arrays.stream(jsonString.split(System.lineSeparator())) - .filter(line -> !line.startsWith("#")) - .count(); + .filter(line -> !line.startsWith("#")) + .count(); } } \ No newline at end of file diff --git a/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/test/BatchJobRepositoryTestConfiguration.java b/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/test/BatchJobRepositoryTestConfiguration.java new file mode 100644 index 000000000..dd4a09fb2 --- /dev/null +++ b/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/test/BatchJobRepositoryTestConfiguration.java @@ -0,0 +1,22 @@ +package uk.ac.ebi.eva.accession.dbsnp2.test; + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Primary; +import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseBuilder; +import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseType; + +import javax.sql.DataSource; + +@Configuration +public class BatchJobRepositoryTestConfiguration { + + @Bean + @Primary + public DataSource dataSource() { + return new EmbeddedDatabaseBuilder() + .setType(EmbeddedDatabaseType.HSQL) + .addScript("org/springframework/batch/core/schema-hsqldb.sql") + .build(); + } +} \ No newline at end of file diff --git a/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/test/BatchTestConfiguration.java b/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/test/BatchTestConfiguration.java index ffa8b1a8c..e88926287 100644 --- a/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/test/BatchTestConfiguration.java +++ b/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/test/BatchTestConfiguration.java @@ -15,29 +15,31 @@ */ package uk.ac.ebi.eva.accession.dbsnp2.test; +import org.springframework.batch.core.Job; import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; +import org.springframework.batch.core.launch.JobLauncher; +import org.springframework.batch.core.repository.JobRepository; import org.springframework.batch.test.JobLauncherTestUtils; -import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; -import org.springframework.boot.autoconfigure.batch.BatchProperties; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Import; -import org.springframework.transaction.PlatformTransactionManager; - import uk.ac.ebi.eva.accession.core.configuration.nonhuman.MongoConfiguration; -import uk.ac.ebi.eva.accession.dbsnp2.configuration.batch.policies.ChunkSizeCompletionPolicyConfiguration; +import uk.ac.ebi.eva.accession.dbsnp2.configuration.InputParametersConfiguration; import uk.ac.ebi.eva.accession.dbsnp2.configuration.batch.flow.ImportDbsnpJsonFlowConfiguration; -import uk.ac.ebi.eva.accession.dbsnp2.configuration.batch.jobs.ImportDbsnpJsonVariantsJobConfiguration; import uk.ac.ebi.eva.accession.dbsnp2.configuration.batch.io.ImportDbsnpJsonVariantsReaderConfiguration; -import uk.ac.ebi.eva.accession.dbsnp2.configuration.batch.steps.ImportDbsnpJsonVariantsStepConfiguration; import uk.ac.ebi.eva.accession.dbsnp2.configuration.batch.io.ImportDbsnpJsonVariantsWriterConfiguration; -import uk.ac.ebi.eva.accession.dbsnp2.configuration.InputParametersConfiguration; -import uk.ac.ebi.eva.accession.dbsnp2.configuration.batch.processors.JsonNodeToClusteredVariantProcessorConfiguration; +import uk.ac.ebi.eva.accession.dbsnp2.configuration.batch.jobs.ImportDbsnpJsonVariantsJobConfiguration; import uk.ac.ebi.eva.accession.dbsnp2.configuration.batch.listeners.ListenersConfiguration; +import uk.ac.ebi.eva.accession.dbsnp2.configuration.batch.policies.ChunkSizeCompletionPolicyConfiguration; +import uk.ac.ebi.eva.accession.dbsnp2.configuration.batch.processors.JsonNodeToClusteredVariantProcessorConfiguration; +import uk.ac.ebi.eva.accession.dbsnp2.configuration.batch.steps.ImportDbsnpJsonVariantsStepConfiguration; import uk.ac.ebi.eva.accession.dbsnp2.runner.DbsnpJsonImportVariantsJobLauncherCommandLineRunner; import uk.ac.ebi.eva.commons.batch.job.JobExecutionApplicationListener; +import static uk.ac.ebi.eva.accession.dbsnp2.configuration.BeanNames.IMPORT_DBSNP_JSON_VARIANTS_JOB; + @Configuration @EnableAutoConfiguration @EnableBatchProcessing @@ -54,15 +56,16 @@ DbsnpJsonImportVariantsJobLauncherCommandLineRunner.class}) public class BatchTestConfiguration { - @Autowired - private BatchProperties properties; - - @Autowired - private PlatformTransactionManager platformTransactionManager; + public static final String JOB_IMPORT_DBSNP_JSON_VARIANTS_JOB = "JOB_IMPORT_DBSNP_JSON_VARIANTS_JOB"; - @Bean - public JobLauncherTestUtils jobLauncherTestUtils() { - return new JobLauncherTestUtils(); + @Bean(JOB_IMPORT_DBSNP_JSON_VARIANTS_JOB) + public JobLauncherTestUtils jobLauncherTestUtilsFromMongo(JobLauncher jobLauncher, JobRepository jobRepository, + @Qualifier(IMPORT_DBSNP_JSON_VARIANTS_JOB) Job job) { + JobLauncherTestUtils utils = new JobLauncherTestUtils(); + utils.setJobLauncher(jobLauncher); + utils.setJobRepository(jobRepository); + utils.setJob(job); + return utils; } @Bean diff --git a/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/test/MongoTestConfiguration.java b/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/test/MongoTestConfiguration.java new file mode 100644 index 000000000..63280b6dd --- /dev/null +++ b/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/test/MongoTestConfiguration.java @@ -0,0 +1,102 @@ +/* + * Copyright 2021 EMBL - European Bioinformatics Institute + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package uk.ac.ebi.eva.accession.dbsnp2.test; + +import com.mongodb.ReadPreference; +import com.mongodb.WriteConcern; +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoClients; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.domain.EntityScan; +import org.springframework.boot.test.autoconfigure.data.mongo.AutoConfigureDataMongo; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Primary; +import org.springframework.data.mongodb.MongoDatabaseFactory; +import org.springframework.data.mongodb.config.EnableMongoAuditing; +import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.data.mongodb.core.SimpleMongoClientDatabaseFactory; +import org.springframework.data.mongodb.core.WriteResultChecking; +import org.springframework.data.mongodb.core.convert.DefaultDbRefResolver; +import org.springframework.data.mongodb.core.convert.DefaultMongoTypeMapper; +import org.springframework.data.mongodb.core.convert.MappingMongoConverter; +import org.springframework.data.mongodb.core.convert.MongoCustomConversions; +import org.springframework.data.mongodb.core.mapping.MongoMappingContext; +import org.springframework.data.mongodb.repository.config.EnableMongoRepositories; + +import java.util.Collections; + +@Configuration +@EntityScan(basePackages = {"uk.ac.ebi.eva.accession.core.persistence"}) +@EnableMongoRepositories(basePackages = { + "uk.ac.ebi.eva.accession.core.persistence", + "uk.ac.ebi.ampt2d.commons.accession.persistence.mongodb.repository"}) +@EnableMongoAuditing +@AutoConfigureDataMongo +public class MongoTestConfiguration { + + @Value("${spring.data.mongodb.uri}") + private String mongoUri; + + @Value("${spring.data.mongodb.database}") + private String database; + + @Value("${mongodb.read-preference:primary}") + private String readPreference; + + @Bean + @Primary + public MongoClient mongoClient() { + return MongoClients.create(mongoUri); + } + + @Bean("primaryFactory") + @Primary + public MongoDatabaseFactory mongoDbFactory() { + return new SimpleMongoClientDatabaseFactory(mongoClient(), database); + } + + @Bean + @Primary + public MongoCustomConversions mongoCustomConversions() { + return new MongoCustomConversions(Collections.emptyList()); + } + + @Bean + @Primary + public MongoMappingContext mongoMappingContext(MongoCustomConversions conversions) { + MongoMappingContext context = new MongoMappingContext(); + context.setSimpleTypeHolder(conversions.getSimpleTypeHolder()); + context.setAutoIndexCreation(false); + return context; + } + + @Bean + @Primary + public MongoTemplate mongoTemplate(MongoMappingContext mongoMappingContext, + MongoCustomConversions conversions) { + MappingMongoConverter converter = new MappingMongoConverter( + new DefaultDbRefResolver(mongoDbFactory()), mongoMappingContext); + converter.setCustomConversions(conversions); + converter.setTypeMapper(new DefaultMongoTypeMapper(null)); + converter.afterPropertiesSet(); + MongoTemplate mongoTemplate = new MongoTemplate(mongoDbFactory(), converter); + mongoTemplate.setReadPreference(ReadPreference.valueOf(readPreference)); + mongoTemplate.setWriteConcern(WriteConcern.MAJORITY); + mongoTemplate.setWriteResultChecking(WriteResultChecking.EXCEPTION); + return mongoTemplate; + } +} diff --git a/eva-accession-import-dbsnp2/src/test/resources/application.properties b/eva-accession-import-dbsnp2/src/test/resources/application.properties index 492409f84..d2e66060f 100644 --- a/eva-accession-import-dbsnp2/src/test/resources/application.properties +++ b/eva-accession-import-dbsnp2/src/test/resources/application.properties @@ -13,10 +13,7 @@ parameters.chunkSize=5 parameters.forceRestart=false # MongoDB for storing imported accessions -spring.data.mongodb.host=localhost -spring.data.mongodb.port=27017 spring.data.mongodb.database=admin -spring.data.mongodb.password= mongodb.read-preference=primaryPreferred diff --git a/eva-accession-import-dbsnp2/src/test/resources/test-dbsnp-json-import.properties b/eva-accession-import-dbsnp2/src/test/resources/test-dbsnp-json-import.properties deleted file mode 100644 index e69de29bb..000000000 diff --git a/eva-accession-pipeline/pom.xml b/eva-accession-pipeline/pom.xml index 26a0498b9..34e3f8e38 100644 --- a/eva-accession-pipeline/pom.xml +++ b/eva-accession-pipeline/pom.xml @@ -7,7 +7,7 @@ eva-accession uk.ac.ebi.eva - 0.6.58-SNAPSHOT + ${revision} eva-accession-pipeline @@ -32,14 +32,37 @@ postgresql + + com.google.code.gson + gson + + + + uk.ac.ebi.eva + eva-accession-core + ${revision} + test-jar + test + + org.springframework.batch spring-batch-test test - com.lordofthejars - nosqlunit-mongodb + org.springframework.boot + spring-boot-testcontainers + test + + + org.testcontainers + mongodb + test + + + org.testcontainers + junit-jupiter test @@ -91,8 +114,4 @@ - - localhost - - diff --git a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/batch/io/AccessionReportWriter.java b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/batch/io/AccessionReportWriter.java index e209dfb0d..d1e5d867e 100644 --- a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/batch/io/AccessionReportWriter.java +++ b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/batch/io/AccessionReportWriter.java @@ -22,13 +22,12 @@ import org.springframework.batch.item.ItemStreamException; import uk.ac.ebi.ampt2d.commons.accession.core.models.AccessionWrapper; import uk.ac.ebi.ampt2d.commons.accession.core.models.GetOrCreateAccessionWrapper; - -import uk.ac.ebi.eva.accession.core.model.ISubmittedVariant; -import uk.ac.ebi.eva.accession.core.model.SubmittedVariant; +import uk.ac.ebi.eva.accession.core.batch.io.FastaSequenceReader; import uk.ac.ebi.eva.accession.core.contig.ContigMapping; import uk.ac.ebi.eva.accession.core.contig.ContigNaming; import uk.ac.ebi.eva.accession.core.contig.ContigSynonyms; -import uk.ac.ebi.eva.accession.core.batch.io.FastaSequenceReader; +import uk.ac.ebi.eva.accession.core.model.ISubmittedVariant; +import uk.ac.ebi.eva.accession.core.model.SubmittedVariant; import uk.ac.ebi.eva.accession.pipeline.batch.processors.ContigToGenbankReplacerProcessor; import uk.ac.ebi.eva.accession.pipeline.batch.tasklets.reportCheck.AccessionWrapperComparator; import uk.ac.ebi.eva.commons.core.models.IVariant; @@ -114,6 +113,7 @@ public void setAccessionPrefix(String accessionPrefix) { /** * We do not load the variants written in previous failed executions, but there might be duplicates hard to avoid, + * * @see AccessionReportWriterTest#resumeWritingWithRepeatedVariant */ public void open(ExecutionContext executionContext) throws ItemStreamException { @@ -132,16 +132,16 @@ public void open(ExecutionContext executionContext) throws ItemStreamException { } else { throw new IllegalStateException( "Can not resume step safely. All temporary files from the previous execution (" - + contigsOutput.getAbsolutePath() + " and " + variantsOutput.getAbsolutePath() - + ") should exist to resume the step. Please delete those files and start a new job."); + + contigsOutput.getAbsolutePath() + " and " + variantsOutput.getAbsolutePath() + + ") should exist to resume the step. Please delete those files and start a new job."); } } else { // we started a new job if (contigsOutput.exists() || variantsOutput.exists()) { throw new IllegalStateException( "A new job was started (did not resume a previous job) but temporary files exist (" - + contigsOutput.getAbsolutePath() + " or " + variantsOutput.getAbsolutePath() - + "). Please delete them and start a new job"); + + contigsOutput.getAbsolutePath() + " or " + variantsOutput.getAbsolutePath() + + "). Please delete them and start a new job"); } else { boolean append = false; this.contigsWriter = new BufferedWriter(new FileWriter(this.contigsOutput, append)); @@ -165,8 +165,8 @@ private void loadContigMappingFromTemporaryFile(File contigMappingFile) throws I String[] contigColumns = line.split("\t"); if (contigColumns.length != 2) { throw new IllegalStateException("Temporary file " + contigMappingFile.getAbsolutePath() - + " doesn't have the expected format. Please delete it and " - + "start a new job."); + + " doesn't have the expected format. Please delete it and " + + "start a new job."); } String inputContig = contigColumns[0]; String insdcContigAccession = contigColumns[1]; @@ -186,16 +186,16 @@ public void close() throws ItemStreamException { if (!duplicatedInputContigsToInsdc.isEmpty()) { logger.error( "The same chromosome (in the original input) was replaced by several INSDC contig accessions. " - + "This happened for: " + duplicatedInputContigsToInsdc.toString()); + + "This happened for: " + duplicatedInputContigsToInsdc.toString()); } if (!duplicatedInsdcToInputContigs.isEmpty()) { logger.error("The same INSDC contig accessions replaced several input chromosomes. This happened for: " - + duplicatedInsdcToInputContigs.toString()); + + duplicatedInsdcToInputContigs.toString()); } if (!duplicatedInputContigsToInsdc.isEmpty() || !duplicatedInsdcToInputContigs.isEmpty()) { throw new IllegalStateException( "Contig replacement was done but the replacements were not unique. See errors above for " - + "details"); + + "details"); } } catch (IOException e) { throw new ItemStreamException(e); @@ -206,7 +206,7 @@ public void write(List originalVariantsWithInsdcContigs, List> accessionedVariants) throws IOException { if (variantsWriter == null) { throw new IOException("The file " + variantsOutput + " was not opened properly. Hint: Check that the code " - + "called " + this.getClass().getSimpleName() + "::open"); + + "called " + this.getClass().getSimpleName() + "::open"); } updateChromosomeMappings(originalVariantsWithInsdcContigs); List> denormalizedVariants = denormalizeVariants( @@ -225,7 +225,7 @@ private void updateChromosomeMappings(List originalVariantsW String currentInsdcContig = variantWithContig.getChromosome(); String previousInsdcReplacementForOriginalChromosome = inputContigsToInsdc.put(originalChromosome, - currentInsdcContig); + currentInsdcContig); if (previousInsdcReplacementForOriginalChromosome == null) { // there was no previous entry, so this is not present in the contigs file: write it contigsWriter.write(originalChromosome + "\t" + currentInsdcContig); @@ -250,17 +250,17 @@ private void updateChromosomeMappings(List originalVariantsW private String getOriginalChromosome(IVariant variant) { Set originalChromosomes = variant.getSourceEntries() - .stream() - .map(se -> se.getAttributes().get( - ContigToGenbankReplacerProcessor.ORIGINAL_CHROMOSOME)) - .collect(Collectors.toSet()); + .stream() + .map(se -> se.getAttributes().get( + ContigToGenbankReplacerProcessor.ORIGINAL_CHROMOSOME)) + .collect(Collectors.toSet()); if (originalChromosomes.size() != 1) { throw new IllegalStateException( "Bug detected: Multiple original chromosomes were found to be associated with the same variant " - + variant.toString() + ". The attributes had the next list of original chromosomes: [" - + String.join(", ", originalChromosomes) + "]. Contig '" + variant.getChromosome() - + "' was used as replacement."); + + variant.toString() + ". The attributes had the next list of original chromosomes: [" + + String.join(", ", originalChromosomes) + "]. Contig '" + variant.getChromosome() + + "' was used as replacement."); } return originalChromosomes.iterator().next(); } @@ -270,7 +270,7 @@ private List> denorm List> denormalizedAccessions = new ArrayList<>(); for (AccessionWrapper accession : accessions) { denormalizedAccessions.add(new AccessionWrapper<>(accession.getAccession(), accession.getHash(), - denormalizeVariant(accession.getData()))); + denormalizeVariant(accession.getData()))); } return denormalizedAccessions; } @@ -281,7 +281,7 @@ private ISubmittedVariant denormalizeVariant(ISubmittedVariant normalizedVariant return createVariantWithContextBase(normalizedVariant); } else { throw new IllegalArgumentException("Contig '" + normalizedVariant.getContig() - + "' does not appear in the FASTA file "); + + "' does not appear in the FASTA file "); } } else { return normalizedVariant; @@ -294,30 +294,30 @@ private ISubmittedVariant createVariantWithContextBase(ISubmittedVariant normali long oldStart = normalizedVariant.getStart(); ImmutableTriple contextNucleotideInfo = fastaSequenceReader.getContextNucleotideAndNewStart(normalizedVariant.getContig(), oldStart, - oldReference, oldAlternate); + oldReference, oldAlternate); return new SubmittedVariant(normalizedVariant.getReferenceSequenceAccession(), - normalizedVariant.getTaxonomyAccession(), - normalizedVariant.getProjectAccession(), - normalizedVariant.getContig(), - contextNucleotideInfo.getLeft(), - contextNucleotideInfo.getMiddle(), - contextNucleotideInfo.getRight(), - normalizedVariant.getClusteredVariantAccession(), - normalizedVariant.isSupportedByEvidence(), - normalizedVariant.isAssemblyMatch(), - normalizedVariant.isAllelesMatch(), - normalizedVariant.isValidated(), - normalizedVariant.getCreatedDate()); + normalizedVariant.getTaxonomyAccession(), + normalizedVariant.getProjectAccession(), + normalizedVariant.getContig(), + contextNucleotideInfo.getLeft(), + contextNucleotideInfo.getMiddle(), + contextNucleotideInfo.getRight(), + normalizedVariant.getClusteredVariantAccession(), + normalizedVariant.isSupportedByEvidence(), + normalizedVariant.isAssemblyMatch(), + normalizedVariant.isAllelesMatch(), + normalizedVariant.isValidated(), + normalizedVariant.getCreatedDate()); } /** * Replace the contig using the requested contig naming and write the variant to the output file. - * + *

* Note how this is done after the sorting (using {@link AccessionWrapperComparator}) because the mappings we * passed to it are mappings from the input naming to INSDC, not from input naming to requested output naming. - * + *

* Also, we have to get the equivalent of the original chromosome (not the INSDC contig) because we will mostly use * {@link ContigNaming#NO_REPLACEMENT} and it means "do not replace the submitter's original chromosome". */ @@ -327,12 +327,12 @@ private void writeSortedVariant(AccessionWrapper variants) throws Exception { + public void write(Chunk variants) throws Exception { if (!variants.isEmpty()) { - List submittedVariants = variants.stream().map(variantConverter::convert) + List submittedVariants = variants.getItems().stream().map(variantConverter::convert) .collect(Collectors.toList()); List> accessions = service.getOrCreate(submittedVariants, jobExecution.getJobId().toString()); metricCompute.addCount(AccessioningMetric.SUBMITTED_VARIANTS, variants.size()); metricCompute.addCount(AccessioningMetric.ACCESSIONED_VARIANTS, accessions.size()); - accessionReportWriter.write(variants, accessions); + accessionReportWriter.write(variants.getItems(), accessions); checkCountsMatch(submittedVariants, accessions); } } @@ -77,28 +78,28 @@ void checkCountsMatch(List variants, List> accessions) { if (variants.size() != accessions.size()) { Set accessionedVariants = accessions.stream() - .map(AccessionWrapper::getData) - .map(this::getSubmittedVariantWithoutClusteredVariantAccession) - .collect(Collectors.toSet()); + .map(AccessionWrapper::getData) + .map(this::getSubmittedVariantWithoutClusteredVariantAccession) + .collect(Collectors.toSet()); HashSet distinctVariants = new HashSet<>(variants); int duplicateCount = variants.size() - distinctVariants.size(); metricCompute.addCount(AccessioningMetric.DISTINCT_VARIANTS, distinctVariants.size()); metricCompute.addCount(AccessioningMetric.DUPLICATE_VARIANTS, duplicateCount); if (duplicateCount != 0) { logger.warn("A variant chunk contains {} repeated variants. This is not an error, but please check " + - "it's expected.", duplicateCount); + "it's expected.", duplicateCount); } Set variantsWithoutAccession = distinctVariants.stream() - .filter(v -> !accessionedVariants - .contains(v)) - .collect(Collectors.toSet()); + .filter(v -> !accessionedVariants + .contains(v)) + .collect(Collectors.toSet()); metricCompute.addCount(AccessioningMetric.DISCARDED_VARIANTS, variantsWithoutAccession.size()); if (variantsWithoutAccession.size() != 0) { logger.error("A problem occurred while accessioning a chunk. Total num variants = {}, distinct = {}, " + - "duplicate = {}, accessioned = {}, not accessioned = {}", - variants.size(), distinctVariants.size(), duplicateCount, accessionedVariants.size(), - variantsWithoutAccession.size()); + "duplicate = {}, accessioned = {}, not accessioned = {}", + variants.size(), distinctVariants.size(), duplicateCount, accessionedVariants.size(), + variantsWithoutAccession.size()); logger.error("The non-accessioned variants are: {}", variantsWithoutAccession.toString()); throw new IllegalStateException( @@ -109,18 +110,18 @@ void checkCountsMatch(List variants, private ISubmittedVariant getSubmittedVariantWithoutClusteredVariantAccession(ISubmittedVariant submittedVariant) { return new SubmittedVariant(submittedVariant.getReferenceSequenceAccession(), - submittedVariant.getTaxonomyAccession(), - submittedVariant.getProjectAccession(), - submittedVariant.getContig(), - submittedVariant.getStart(), - submittedVariant.getReferenceAllele(), - submittedVariant.getAlternateAllele(), - null, - submittedVariant.isSupportedByEvidence(), - submittedVariant.isAssemblyMatch(), - submittedVariant.isAllelesMatch(), - submittedVariant.isValidated(), - submittedVariant.getCreatedDate()); + submittedVariant.getTaxonomyAccession(), + submittedVariant.getProjectAccession(), + submittedVariant.getContig(), + submittedVariant.getStart(), + submittedVariant.getReferenceAllele(), + submittedVariant.getAlternateAllele(), + null, + submittedVariant.isSupportedByEvidence(), + submittedVariant.isAssemblyMatch(), + submittedVariant.isAllelesMatch(), + submittedVariant.isValidated(), + submittedVariant.getCreatedDate()); } @Override diff --git a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/batch/io/DuplicateSSAccQCWriter.java b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/batch/io/DuplicateSSAccQCWriter.java index b9a26a4bb..84defd8dd 100644 --- a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/batch/io/DuplicateSSAccQCWriter.java +++ b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/batch/io/DuplicateSSAccQCWriter.java @@ -1,9 +1,10 @@ package uk.ac.ebi.eva.accession.pipeline.batch.io; -import gherkin.deps.com.google.gson.Gson; -import gherkin.deps.com.google.gson.GsonBuilder; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.batch.item.Chunk; import org.springframework.batch.item.ExecutionContext; import org.springframework.batch.item.ItemStreamException; import org.springframework.batch.item.ItemStreamWriter; @@ -48,7 +49,7 @@ public void close() throws ItemStreamException { } @Override - public void write(List> listOfDuplicateSSAccQCResultLists) throws Exception { + public void write(Chunk> listOfDuplicateSSAccQCResultLists) throws Exception { for (List duplicateSSAccQCResultList : listOfDuplicateSSAccQCResultLists) { if (duplicateSSAccQCResultList != null && !duplicateSSAccQCResultList.isEmpty()) { appendToFile(duplicateSSAccQCResultList); diff --git a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/batch/policies/InvalidVariantSkipPolicy.java b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/batch/policies/InvalidVariantSkipPolicy.java index f6b6c5c3b..cb5e4984d 100644 --- a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/batch/policies/InvalidVariantSkipPolicy.java +++ b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/batch/policies/InvalidVariantSkipPolicy.java @@ -31,7 +31,7 @@ public class InvalidVariantSkipPolicy implements SkipPolicy { @Override - public boolean shouldSkip(Throwable exception, int skipCount) throws SkipLimitExceededException { + public boolean shouldSkip(Throwable exception, long skipCount) throws SkipLimitExceededException { if (exception instanceof FlatFileParseException && (exception.getCause() instanceof NonVariantException || exception.getCause() instanceof IncompleteInformationException)) { diff --git a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/RunnerConfiguration.java b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/RunnerConfiguration.java index 517525c34..6b60049ff 100644 --- a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/RunnerConfiguration.java +++ b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/RunnerConfiguration.java @@ -16,16 +16,10 @@ */ package uk.ac.ebi.eva.accession.pipeline.configuration; -import org.springframework.batch.core.configuration.annotation.BatchConfigurer; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; - -import uk.ac.ebi.eva.commons.batch.configuration.SpringBoot1CompatibilityConfiguration; import uk.ac.ebi.eva.commons.batch.job.JobExecutionApplicationListener; -import javax.persistence.EntityManagerFactory; -import javax.sql.DataSource; - @Configuration public class RunnerConfiguration { @@ -33,11 +27,4 @@ public class RunnerConfiguration { public JobExecutionApplicationListener jobExecutionApplicationListener() { return new JobExecutionApplicationListener(); } - - @Bean - public BatchConfigurer configurer(DataSource dataSource, EntityManagerFactory entityManagerFactory) - throws Exception { - return SpringBoot1CompatibilityConfiguration.getSpringBoot1CompatibleBatchConfigurer(dataSource, - entityManagerFactory); - } } diff --git a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/io/AccessionWriterConfiguration.java b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/io/AccessionWriterConfiguration.java index 34baa7fe7..8071acec2 100644 --- a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/io/AccessionWriterConfiguration.java +++ b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/io/AccessionWriterConfiguration.java @@ -18,17 +18,16 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Import; - -import uk.ac.ebi.eva.accession.core.service.nonhuman.SubmittedVariantAccessioningService; +import uk.ac.ebi.eva.accession.core.batch.io.FastaSynonymSequenceReader; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.SubmittedVariantAccessioningConfiguration; import uk.ac.ebi.eva.accession.core.contig.ContigMapping; -import uk.ac.ebi.eva.accession.core.batch.io.FastaSynonymSequenceReader; -import uk.ac.ebi.eva.accession.pipeline.configuration.InputParametersConfiguration; +import uk.ac.ebi.eva.accession.core.service.nonhuman.SubmittedVariantAccessioningService; import uk.ac.ebi.eva.accession.pipeline.batch.io.AccessionReportWriter; import uk.ac.ebi.eva.accession.pipeline.batch.io.AccessionWriter; -import uk.ac.ebi.eva.accession.pipeline.parameters.InputParameters; import uk.ac.ebi.eva.accession.pipeline.batch.processors.VariantConverter; +import uk.ac.ebi.eva.accession.pipeline.configuration.InputParametersConfiguration; import uk.ac.ebi.eva.accession.pipeline.configuration.batch.listeners.ListenersConfiguration; +import uk.ac.ebi.eva.accession.pipeline.parameters.InputParameters; import uk.ac.ebi.eva.metrics.metric.MetricCompute; import java.io.File; diff --git a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/io/VcfReaderConfiguration.java b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/io/VcfReaderConfiguration.java index 2a8c75996..0123827b3 100644 --- a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/io/VcfReaderConfiguration.java +++ b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/io/VcfReaderConfiguration.java @@ -21,14 +21,13 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Import; - import uk.ac.ebi.eva.accession.pipeline.configuration.InputParametersConfiguration; import uk.ac.ebi.eva.accession.pipeline.parameters.InputParameters; +import uk.ac.ebi.eva.commons.batch.io.AggregatedVcfReader; import uk.ac.ebi.eva.commons.batch.io.UnwindingItemStreamReader; +import uk.ac.ebi.eva.commons.batch.io.VcfReader; import uk.ac.ebi.eva.commons.core.models.Aggregation; import uk.ac.ebi.eva.commons.core.models.pipeline.Variant; -import uk.ac.ebi.eva.commons.batch.io.AggregatedVcfReader; -import uk.ac.ebi.eva.commons.batch.io.VcfReader; import java.io.File; import java.io.IOException; diff --git a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/DuplicateSSAccQCJobConfiguration.java b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/DuplicateSSAccQCJobConfiguration.java index 7ee57a6c4..b3f570c39 100644 --- a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/DuplicateSSAccQCJobConfiguration.java +++ b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/DuplicateSSAccQCJobConfiguration.java @@ -3,8 +3,9 @@ import org.springframework.batch.core.Job; import org.springframework.batch.core.Step; import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; -import org.springframework.batch.core.configuration.annotation.JobBuilderFactory; +import org.springframework.batch.core.job.builder.JobBuilder; import org.springframework.batch.core.launch.support.RunIdIncrementer; +import org.springframework.batch.core.repository.JobRepository; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; @@ -22,8 +23,8 @@ public class DuplicateSSAccQCJobConfiguration { private Step duplicateSSAccQCStep; @Bean(DUPLICATE_SS_ACC_QC_JOB) - public Job duplicateSSAccQCJob(JobBuilderFactory jobBuilderFactory) { - return jobBuilderFactory.get(DUPLICATE_SS_ACC_QC_JOB) + public Job duplicateSSAccQCJob(JobRepository jobRepository) { + return new JobBuilder(DUPLICATE_SS_ACC_QC_JOB, jobRepository) .incrementer(new RunIdIncrementer()) .start(duplicateSSAccQCStep) .build(); diff --git a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/QCSubsnpAccessionsJobConfiguration.java b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/QCSubsnpAccessionsJobConfiguration.java index 5caafdd24..6ee75487b 100644 --- a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/QCSubsnpAccessionsJobConfiguration.java +++ b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/QCSubsnpAccessionsJobConfiguration.java @@ -3,8 +3,9 @@ import org.springframework.batch.core.Job; import org.springframework.batch.core.Step; import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; -import org.springframework.batch.core.configuration.annotation.JobBuilderFactory; +import org.springframework.batch.core.job.builder.JobBuilder; import org.springframework.batch.core.launch.support.RunIdIncrementer; +import org.springframework.batch.core.repository.JobRepository; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; @@ -21,8 +22,8 @@ public class QCSubsnpAccessionsJobConfiguration { private Step qcSubsnpAccessionStep; @Bean(QC_SUBSNP_ACCESSION_JOB) - public Job qcSubsnpAccessionJob(JobBuilderFactory jobBuilderFactory) { - return jobBuilderFactory.get(QC_SUBSNP_ACCESSION_JOB) + public Job qcSubsnpAccessionJob(JobRepository jobRepository) { + return new JobBuilder(QC_SUBSNP_ACCESSION_JOB, jobRepository) .incrementer(new RunIdIncrementer()) .start(qcSubsnpAccessionStep) .build(); diff --git a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/SSAccessionRecoveryJobConfiguration.java b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/SSAccessionRecoveryJobConfiguration.java index 25a2669d2..878ed8ad9 100644 --- a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/SSAccessionRecoveryJobConfiguration.java +++ b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/SSAccessionRecoveryJobConfiguration.java @@ -4,8 +4,9 @@ import org.springframework.batch.core.JobExecutionListener; import org.springframework.batch.core.Step; import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; -import org.springframework.batch.core.configuration.annotation.JobBuilderFactory; +import org.springframework.batch.core.job.builder.JobBuilder; import org.springframework.batch.core.launch.support.RunIdIncrementer; +import org.springframework.batch.core.repository.JobRepository; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; @@ -28,8 +29,8 @@ public class SSAccessionRecoveryJobConfiguration { private JobExecutionListener ssAccessionRecoveryJobListener; @Bean(SS_ACCESSION_RECOVERY_JOB) - public Job createSSAccessionRecoveryJob(JobBuilderFactory jobBuilderFactory) { - return jobBuilderFactory.get(SS_ACCESSION_RECOVERY_JOB) + public Job createSSAccessionRecoveryJob(JobRepository jobRepository) { + return new JobBuilder(SS_ACCESSION_RECOVERY_JOB, jobRepository) .incrementer(new RunIdIncrementer()) .start(ssAccessionRecoveryStep) .listener(ssAccessionRecoveryJobListener) diff --git a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/SubsnpAccessionsJobConfiguration.java b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/SubsnpAccessionsJobConfiguration.java index 9907ff169..43290594d 100644 --- a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/SubsnpAccessionsJobConfiguration.java +++ b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/SubsnpAccessionsJobConfiguration.java @@ -20,8 +20,9 @@ import org.springframework.batch.core.JobExecutionListener; import org.springframework.batch.core.Step; import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; -import org.springframework.batch.core.configuration.annotation.JobBuilderFactory; +import org.springframework.batch.core.job.builder.JobBuilder; import org.springframework.batch.core.launch.support.RunIdIncrementer; +import org.springframework.batch.core.repository.JobRepository; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; @@ -54,8 +55,8 @@ public class SubsnpAccessionsJobConfiguration { private JobExecutionListener subsnpAccessionJobListener; @Bean(SUBSNP_ACCESSION_JOB) - public Job subsnpAccessionJob(JobBuilderFactory jobBuilderFactory) { - return jobBuilderFactory.get(SUBSNP_ACCESSION_JOB) + public Job subsnpAccessionJob(JobRepository jobRepository) { + return new JobBuilder(SUBSNP_ACCESSION_JOB, jobRepository) .incrementer(new RunIdIncrementer()) .start(subsnpAccessionStep) .next(accessioningShutdownStep) diff --git a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/processors/VariantProcessorConfiguration.java b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/processors/VariantProcessorConfiguration.java index f164b54aa..56d476846 100644 --- a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/processors/VariantProcessorConfiguration.java +++ b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/processors/VariantProcessorConfiguration.java @@ -23,11 +23,10 @@ import org.springframework.batch.item.support.CompositeItemProcessor; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; - import uk.ac.ebi.eva.accession.core.contig.ContigMapping; -import uk.ac.ebi.eva.accession.pipeline.parameters.InputParameters; import uk.ac.ebi.eva.accession.pipeline.batch.processors.ContigToGenbankReplacerProcessor; import uk.ac.ebi.eva.accession.pipeline.batch.processors.ExcludeStructuralVariantsProcessor; +import uk.ac.ebi.eva.accession.pipeline.parameters.InputParameters; import uk.ac.ebi.eva.commons.core.models.IVariant; import java.util.Arrays; diff --git a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/AccessioningShutdownStepConfiguration.java b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/AccessioningShutdownStepConfiguration.java index 75f083f2d..ccad6721b 100644 --- a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/AccessioningShutdownStepConfiguration.java +++ b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/AccessioningShutdownStepConfiguration.java @@ -2,10 +2,12 @@ import org.springframework.batch.core.Step; import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; -import org.springframework.batch.core.configuration.annotation.StepBuilderFactory; +import org.springframework.batch.core.repository.JobRepository; +import org.springframework.batch.core.step.builder.StepBuilder; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.transaction.PlatformTransactionManager; import uk.ac.ebi.eva.accession.core.service.nonhuman.SubmittedVariantAccessioningService; import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.ACCESSIONING_SHUTDOWN_STEP; @@ -17,12 +19,12 @@ public class AccessioningShutdownStepConfiguration { private SubmittedVariantAccessioningService submittedVariantAccessioningService; @Bean(ACCESSIONING_SHUTDOWN_STEP) - public Step accessioningShutDownStep(StepBuilderFactory stepBuilderFactory) { - return stepBuilderFactory.get(ACCESSIONING_SHUTDOWN_STEP) + public Step accessioningShutDownStep(JobRepository jobRepository, PlatformTransactionManager transactionManager) { + return new StepBuilder(ACCESSIONING_SHUTDOWN_STEP, jobRepository) .tasklet((contribution, chunkContext) -> { submittedVariantAccessioningService.shutDownAccessionGenerator(); return null; - }) + }, transactionManager) .build(); } } \ No newline at end of file diff --git a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/BuildReportStepConfiguration.java b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/BuildReportStepConfiguration.java index 090273e2c..e7fe6d462 100644 --- a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/BuildReportStepConfiguration.java +++ b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/BuildReportStepConfiguration.java @@ -18,17 +18,17 @@ import org.springframework.batch.core.Step; import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; -import org.springframework.batch.core.configuration.annotation.StepBuilderFactory; +import org.springframework.batch.core.repository.JobRepository; +import org.springframework.batch.core.step.builder.StepBuilder; import org.springframework.batch.core.step.tasklet.TaskletStep; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; - -import uk.ac.ebi.eva.accession.pipeline.parameters.InputParameters; +import org.springframework.transaction.PlatformTransactionManager; import uk.ac.ebi.eva.accession.pipeline.batch.tasklets.buildReport.BuildReportTasklet; +import uk.ac.ebi.eva.accession.pipeline.parameters.InputParameters; import java.io.File; -import java.io.IOException; import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.BUILD_REPORT_STEP; @@ -40,11 +40,11 @@ public class BuildReportStepConfiguration { private InputParameters inputParameters; @Bean(BUILD_REPORT_STEP) - public Step buildReportStep(StepBuilderFactory stepBuilderFactory) throws IOException { + public Step buildReportStep(JobRepository jobRepository, PlatformTransactionManager transactionManager) { BuildReportTasklet tasklet = new BuildReportTasklet(new File(inputParameters.getOutputVcf())); - TaskletStep step = stepBuilderFactory.get(BUILD_REPORT_STEP) - .tasklet(tasklet) - .build(); + TaskletStep step = new StepBuilder(BUILD_REPORT_STEP, jobRepository) + .tasklet(tasklet, transactionManager) + .build(); return step; } } diff --git a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/DuplicateSSAccQCStepConfiguration.java b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/DuplicateSSAccQCStepConfiguration.java index 0364100a9..75a9cabde 100644 --- a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/DuplicateSSAccQCStepConfiguration.java +++ b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/DuplicateSSAccQCStepConfiguration.java @@ -2,7 +2,8 @@ import org.springframework.batch.core.Step; import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; -import org.springframework.batch.core.configuration.annotation.StepBuilderFactory; +import org.springframework.batch.core.repository.JobRepository; +import org.springframework.batch.core.step.builder.StepBuilder; import org.springframework.batch.core.step.tasklet.TaskletStep; import org.springframework.batch.item.ItemProcessor; import org.springframework.batch.item.ItemStreamReader; @@ -11,6 +12,7 @@ import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.transaction.PlatformTransactionManager; import uk.ac.ebi.eva.accession.pipeline.batch.io.DuplicateSSAccQCResult; import java.util.List; @@ -37,11 +39,11 @@ public class DuplicateSSAccQCStepConfiguration { private ItemWriter> duplicateSSAccQCWriter; @Bean(DUPLICATE_SS_ACC_QC_STEP) - public Step duplicateSSAccQCStep(StepBuilderFactory stepBuilderFactory) { - TaskletStep step = stepBuilderFactory.get(DUPLICATE_SS_ACC_QC_STEP) + public Step duplicateSSAccQCStep(JobRepository jobRepository, PlatformTransactionManager transactionManager) { + TaskletStep step = new StepBuilder(DUPLICATE_SS_ACC_QC_STEP, jobRepository) // hardcoded the chunk size as 1, as the reader takes care of accumulating // and sending the chunk size (defined in properties file) elements to the processor - ., List>chunk(1) + ., List>chunk(1, transactionManager) .reader(ssAccFileReader) .processor(duplicateSSAccQCProcessor) .writer(duplicateSSAccQCWriter) diff --git a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/QCSubsnpAccessionsStepConfiguration.java b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/QCSubsnpAccessionsStepConfiguration.java index 3af6a5eeb..6fe42f886 100644 --- a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/QCSubsnpAccessionsStepConfiguration.java +++ b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/QCSubsnpAccessionsStepConfiguration.java @@ -18,13 +18,15 @@ import org.springframework.batch.core.Step; import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; -import org.springframework.batch.core.configuration.annotation.StepBuilderFactory; +import org.springframework.batch.core.repository.JobRepository; +import org.springframework.batch.core.step.builder.StepBuilder; import org.springframework.batch.core.step.tasklet.TaskletStep; import org.springframework.batch.item.ItemStreamReader; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.transaction.PlatformTransactionManager; import uk.ac.ebi.eva.accession.core.batch.io.AccessionedVcfLineMapper; import uk.ac.ebi.eva.accession.core.contig.ContigMapping; import uk.ac.ebi.eva.accession.pipeline.batch.tasklets.reportCheck.ReportCheckTasklet; @@ -62,11 +64,11 @@ public ItemStreamReader reportReader() throws IOException { } @Bean(QC_SUBSNP_ACCESSION_STEP) - public Step qcSubsnpAccessionStep(StepBuilderFactory stepBuilderFactory) throws IOException { + public Step qcSubsnpAccessionStep(JobRepository jobRepository, PlatformTransactionManager transactionManager) throws IOException { ReportCheckTasklet tasklet = new ReportCheckTasklet(inputReader, reportReader(), inputParameters.getChunkSize() * 2, contigMapping); - TaskletStep step = stepBuilderFactory.get(QC_SUBSNP_ACCESSION_STEP) - .tasklet(tasklet) + TaskletStep step = new StepBuilder(QC_SUBSNP_ACCESSION_STEP, jobRepository) + .tasklet(tasklet, transactionManager) .build(); return step; } diff --git a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/SSAccessionRecoveryStepConfiguration.java b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/SSAccessionRecoveryStepConfiguration.java index 758d75616..a87432c36 100644 --- a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/SSAccessionRecoveryStepConfiguration.java +++ b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/SSAccessionRecoveryStepConfiguration.java @@ -2,11 +2,13 @@ import org.springframework.batch.core.Step; import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; -import org.springframework.batch.core.configuration.annotation.StepBuilderFactory; +import org.springframework.batch.core.repository.JobRepository; +import org.springframework.batch.core.step.builder.StepBuilder; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.transaction.PlatformTransactionManager; import uk.ac.ebi.eva.accession.pipeline.batch.recovery.SSAccessionRecoveryService; import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.SS_ACCESSION_RECOVERY_SERVICE; @@ -20,12 +22,12 @@ public class SSAccessionRecoveryStepConfiguration { private SSAccessionRecoveryService SSAccessionRecoveryService; @Bean(SS_ACCESSION_RECOVERY_STEP) - public Step ssAccessionRecoveryStep(StepBuilderFactory stepBuilderFactory) { - return stepBuilderFactory.get(SS_ACCESSION_RECOVERY_STEP) + public Step ssAccessionRecoveryStep(JobRepository jobRepository, PlatformTransactionManager transactionManager) { + return new StepBuilder(SS_ACCESSION_RECOVERY_STEP, jobRepository) .tasklet((contribution, chunkContext) -> { SSAccessionRecoveryService.runRecoveryForCategorySS(); return null; - }) + }, transactionManager) .build(); } } diff --git a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/SubsnpAccessionsStepConfiguration.java b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/SubsnpAccessionsStepConfiguration.java index 6ef6c30b4..8f4902761 100644 --- a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/SubsnpAccessionsStepConfiguration.java +++ b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/SubsnpAccessionsStepConfiguration.java @@ -19,7 +19,8 @@ import org.springframework.batch.core.Step; import org.springframework.batch.core.StepExecutionListener; import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; -import org.springframework.batch.core.configuration.annotation.StepBuilderFactory; +import org.springframework.batch.core.repository.JobRepository; +import org.springframework.batch.core.step.builder.StepBuilder; import org.springframework.batch.core.step.tasklet.TaskletStep; import org.springframework.batch.item.ItemProcessor; import org.springframework.batch.item.ItemReader; @@ -29,6 +30,7 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.retry.backoff.ExponentialBackOffPolicy; +import org.springframework.transaction.PlatformTransactionManager; import uk.ac.ebi.ampt2d.commons.accession.core.exceptions.MissingUnsavedAccessionsException; import uk.ac.ebi.eva.accession.pipeline.batch.io.AccessionWriter; import uk.ac.ebi.eva.accession.pipeline.batch.policies.InvalidVariantSkipPolicy; @@ -65,10 +67,10 @@ public class SubsnpAccessionsStepConfiguration { private InvalidVariantSkipPolicy invalidVariantSkipPolicy; @Bean(SUBSNP_ACCESSION_STEP) - public Step subsnpAccessionStep(StepBuilderFactory stepBuilderFactory, + public Step subsnpAccessionStep(JobRepository jobRepository, PlatformTransactionManager transactionManager, SimpleCompletionPolicy chunkSizeCompletionPolicy) { - TaskletStep step = stepBuilderFactory.get(SUBSNP_ACCESSION_STEP) - .chunk(chunkSizeCompletionPolicy) + TaskletStep step = new StepBuilder(SUBSNP_ACCESSION_STEP, jobRepository) + .chunk(chunkSizeCompletionPolicy, transactionManager) .reader(variantReader) .processor(variantProcessor) .writer(accessionWriter) diff --git a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/metric/AccessioningMetricCompute.java b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/metric/AccessioningMetricCompute.java index 16ec97902..4063888cc 100644 --- a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/metric/AccessioningMetricCompute.java +++ b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/metric/AccessioningMetricCompute.java @@ -1,7 +1,8 @@ package uk.ac.ebi.eva.accession.pipeline.metric; -import org.codehaus.jettison.json.JSONException; -import org.codehaus.jettison.json.JSONObject; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; import org.springframework.web.client.RestTemplate; import uk.ac.ebi.eva.metrics.count.CountServiceParameters; import uk.ac.ebi.eva.metrics.metric.BaseMetricCompute; @@ -11,6 +12,8 @@ import java.util.stream.Collectors; public class AccessioningMetricCompute extends BaseMetricCompute { + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + private static final String PROCESS = "accessioning_warehouse_ingestion"; private final String assembly; private final String study; @@ -32,12 +35,13 @@ public List getMetrics() { public String getIdentifier() { try { - JSONObject identifier = new JSONObject(); + ObjectNode identifier = OBJECT_MAPPER.createObjectNode(); identifier.put("assembly", assembly); identifier.put("study", study); - return identifier.toString(); - } catch (JSONException jsonException) { - throw new RuntimeException("Could not create Identifier for Accessioning Counts. Error ", jsonException); + + return OBJECT_MAPPER.writeValueAsString(identifier); + } catch (JsonProcessingException ex) { + throw new RuntimeException("Could not create Identifier for Accessioning Counts. Error ", ex); } } diff --git a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/runner/EvaAccessionJobLauncherCommandLineRunner.java b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/runner/EvaAccessionJobLauncherCommandLineRunner.java index aef346c20..adcc1202c 100644 --- a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/runner/EvaAccessionJobLauncherCommandLineRunner.java +++ b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/runner/EvaAccessionJobLauncherCommandLineRunner.java @@ -24,7 +24,6 @@ import org.springframework.batch.core.JobParametersInvalidException; import org.springframework.batch.core.explore.JobExplorer; import org.springframework.batch.core.launch.JobLauncher; -import org.springframework.batch.core.launch.JobParametersNotFoundException; import org.springframework.batch.core.repository.JobExecutionAlreadyRunningException; import org.springframework.batch.core.repository.JobInstanceAlreadyCompleteException; import org.springframework.batch.core.repository.JobRepository; @@ -32,11 +31,10 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.ExitCodeGenerator; -import org.springframework.boot.autoconfigure.batch.JobLauncherCommandLineRunner; +import org.springframework.boot.autoconfigure.batch.JobLauncherApplicationRunner; import org.springframework.context.ApplicationEventPublisherAware; import org.springframework.stereotype.Component; import org.springframework.util.PatternMatchUtils; - import uk.ac.ebi.eva.accession.core.runner.CommandLineRunnerUtils; import uk.ac.ebi.eva.accession.pipeline.parameters.InputParameters; import uk.ac.ebi.eva.commons.batch.exception.NoJobToExecuteException; @@ -56,7 +54,7 @@ * -The user can restart a job that has been run previously marking the previous execution as failed. */ @Component -public class EvaAccessionJobLauncherCommandLineRunner extends JobLauncherCommandLineRunner implements +public class EvaAccessionJobLauncherCommandLineRunner extends JobLauncherApplicationRunner implements ApplicationEventPublisherAware, ExitCodeGenerator { private static final Logger logger = LoggerFactory.getLogger(EvaAccessionJobLauncherCommandLineRunner.class); @@ -98,9 +96,9 @@ public void setJobs(Collection jobs) { } @Override - public void setJobNames(String jobName) { + public void setJobName(String jobName) { this.jobName = jobName; - super.setJobNames(jobName); + super.setJobName(jobName); } @Override @@ -124,16 +122,15 @@ public void run(String... args) throws JobExecutionException { JobStatusManager.checkIfPropertiesHaveBeenProvided(jobParameters); if (inputParameters.isForceRestart()) { JobExecution previousJobExecution = CommandLineRunnerUtils.getLastJobExecution(jobName, jobExplorer, - jobParameters); + jobParameters); CommandLineRunnerUtils.markPreviousJobAsFailed(jobName, jobRepository, previousJobExecution != null ? previousJobExecution.getJobParameters() : jobParameters); - } - else { + } else { jobParameters = CommandLineRunnerUtils.addRunIDToJobParameters(jobName, jobExplorer, jobParameters); } launchJob(jobParameters); } catch (NoJobToExecuteException | NoParametersHaveBeenPassedException | NoPreviousJobExecutionException - | UnknownJobException | JobParametersInvalidException | JobExecutionAlreadyRunningException e) { + | UnknownJobException | JobParametersInvalidException | JobExecutionAlreadyRunningException e) { logger.error(e.getMessage()); logger.debug("Error trace", e); abnormalExit = true; @@ -155,8 +152,7 @@ private void launchJob(JobParameters jobParameters) throws JobExecutionException @Override protected void execute(Job job, JobParameters jobParameters) throws JobExecutionAlreadyRunningException, - JobRestartException, JobInstanceAlreadyCompleteException, JobParametersInvalidException, - JobParametersNotFoundException { + JobRestartException, JobInstanceAlreadyCompleteException, JobParametersInvalidException { logger.info("Running job '" + jobName + "' with parameters: " + jobParameters); super.execute(job, jobParameters); } diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/batch/io/AccessionReportWriterTest.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/batch/io/AccessionReportWriterTest.java index 3aee49e5f..30e5f654c 100644 --- a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/batch/io/AccessionReportWriterTest.java +++ b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/batch/io/AccessionReportWriterTest.java @@ -15,22 +15,20 @@ */ package uk.ac.ebi.eva.accession.pipeline.batch.io; -import org.junit.Before; -import org.junit.Ignore; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; import org.springframework.batch.item.ExecutionContext; import uk.ac.ebi.ampt2d.commons.accession.core.models.AccessionWrapper; - -import uk.ac.ebi.eva.accession.core.service.GetOrCreateAccessionWrapperCreator; -import uk.ac.ebi.eva.accession.core.model.ISubmittedVariant; -import uk.ac.ebi.eva.accession.core.model.SubmittedVariant; +import uk.ac.ebi.eva.accession.core.batch.io.FastaSequenceReader; +import uk.ac.ebi.eva.accession.core.batch.io.FastaSynonymSequenceReader; import uk.ac.ebi.eva.accession.core.contig.ContigMapping; import uk.ac.ebi.eva.accession.core.contig.ContigNaming; import uk.ac.ebi.eva.accession.core.contig.ContigSynonyms; -import uk.ac.ebi.eva.accession.core.batch.io.FastaSequenceReader; -import uk.ac.ebi.eva.accession.core.batch.io.FastaSynonymSequenceReader; +import uk.ac.ebi.eva.accession.core.model.ISubmittedVariant; +import uk.ac.ebi.eva.accession.core.model.SubmittedVariant; +import uk.ac.ebi.eva.accession.core.service.GetOrCreateAccessionWrapperCreator; +import uk.ac.ebi.eva.accession.core.utils.PipelineTemporaryFolderUtil; import uk.ac.ebi.eva.commons.core.models.pipeline.Variant; import uk.ac.ebi.eva.commons.core.models.pipeline.VariantSourceEntry; @@ -45,7 +43,8 @@ import java.util.Arrays; import java.util.Collections; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; import static uk.ac.ebi.eva.accession.pipeline.batch.processors.ContigToGenbankReplacerProcessor.ORIGINAL_CHROMOSOME; public class AccessionReportWriterTest { @@ -112,14 +111,13 @@ public class AccessionReportWriterTest { private ExecutionContext executionContext; - @Rule - public TemporaryFolder temporaryFolderRule = new TemporaryFolder(); - private ContigMapping contigMapping; - @Before + public PipelineTemporaryFolderUtil temporaryFolderUtil = new PipelineTemporaryFolderUtil(); + + @BeforeEach public void setUp() throws Exception { - output = temporaryFolderRule.newFile(); + output = temporaryFolderUtil.newFile(); variantsOutput = new File(output.getAbsolutePath() + AccessionReportWriter.VARIANTS_FILE_SUFFIX); contigsOutput = new File(output.getAbsolutePath() + AccessionReportWriter.CONTIGS_FILE_SUFFIX); Path fastaPath = Paths.get(AccessionReportWriterTest.class.getResource("/input-files/fasta/mock.fa").toURI()); @@ -127,9 +125,9 @@ public void setUp() throws Exception { new ContigSynonyms(CHROMOSOME_1, "assembled-molecule", "1", CONTIG_1, "refseq_1", "chr1", true), new ContigSynonyms("chr2", "assembled-molecule", "2", GENBANK_2, REFSEQ_2, "chr2", false), new ContigSynonyms(SEQUENCE_NAME_3, "unlocalized-scaffold", "1", GENBANK_3, REFSEQ_3, "chr3_random", - true), + true), new ContigSynonyms(SEQUENCE_NAME_4, "unlocalized-scaffold", "4", GENBANK_4, REFSEQ_4, "chr4_random", - true))); + true))); fastaSequenceReader = new FastaSynonymSequenceReader(contigMapping, fastaPath); executionContext = new ExecutionContext(); } @@ -145,25 +143,25 @@ private void writeVariantWithAccessionHelper(int start, String reference, String Variant variant = buildMockVariant(CONTIG_1, CONTIG_1, start, reference, alternate); SubmittedVariant submittedVariant = new SubmittedVariant("accession", TAXONOMY, "project", CONTIG_1, start, - reference, alternate, CLUSTERED_VARIANT, - SUPPORTED_BY_EVIDENCE, MATCHES_ASSEMBLY, ALLELES_MATCH, - VALIDATED, null); + reference, alternate, CLUSTERED_VARIANT, + SUPPORTED_BY_EVIDENCE, MATCHES_ASSEMBLY, ALLELES_MATCH, + VALIDATED, null); AccessionWrapper accessionWrapper = new AccessionWrapper<>(ACCESSION, "1", - submittedVariant); + submittedVariant); AccessionReportWriter accessionReportWriter = new AccessionReportWriter(output, fastaSequenceReader, - contigMapping, - ContigNaming.NO_REPLACEMENT); + contigMapping, + ContigNaming.NO_REPLACEMENT); accessionReportWriter.open(executionContext); accessionReportWriter.write(Collections.singletonList(variant), - GetOrCreateAccessionWrapperCreator.convertToGetOrCreateAccessionWrapper( - Collections.singletonList(accessionWrapper))); + GetOrCreateAccessionWrapperCreator.convertToGetOrCreateAccessionWrapper( + Collections.singletonList(accessionWrapper))); assertEquals(String.join("\t", CONTIG_1, Integer.toString(denormalizedStart), ACCESSION_PREFIX + ACCESSION, - denormalizedReference, denormalizedAlternate, ".", ".", "."), - getFirstVariantLine(variantsOutput)); + denormalizedReference, denormalizedAlternate, ".", ".", "."), + getFirstVariantLine(variantsOutput)); } public static String getFirstVariantLine(File output) throws IOException { @@ -212,33 +210,33 @@ private void writeAndResumeAndWrite(String originalChromosome1, String contig1, SubmittedVariant submittedVariant = new SubmittedVariant("accession", TAXONOMY, "project", contig1, start1, - REFERENCE, ALTERNATE, CLUSTERED_VARIANT, - SUPPORTED_BY_EVIDENCE, MATCHES_ASSEMBLY, ALLELES_MATCH, - VALIDATED, null); + REFERENCE, ALTERNATE, CLUSTERED_VARIANT, + SUPPORTED_BY_EVIDENCE, MATCHES_ASSEMBLY, ALLELES_MATCH, + VALIDATED, null); AccessionWrapper accessionWrapper = new AccessionWrapper<>(ACCESSION, "1", - submittedVariant); + submittedVariant); AccessionReportWriter accessionReportWriter = new AccessionReportWriter(output, fastaSequenceReader, - contigMapping, - ContigNaming.NO_REPLACEMENT); + contigMapping, + ContigNaming.NO_REPLACEMENT); accessionReportWriter.open(executionContext); accessionReportWriter.write(Collections.singletonList(variant), - GetOrCreateAccessionWrapperCreator.convertToGetOrCreateAccessionWrapper( - Collections.singletonList(accessionWrapper))); + GetOrCreateAccessionWrapperCreator.convertToGetOrCreateAccessionWrapper( + Collections.singletonList(accessionWrapper))); accessionReportWriter.close(); // second writer AccessionReportWriter resumingWriter = new AccessionReportWriter(output, fastaSequenceReader, contigMapping, - ContigNaming.NO_REPLACEMENT); + ContigNaming.NO_REPLACEMENT); variant = buildMockVariant(originalChromosome2, contig2, start2, REFERENCE, ALTERNATE); submittedVariant.setContig(contig2); submittedVariant.setStart(start2); resumingWriter.open(executionContext); resumingWriter.write(Collections.singletonList(variant), - GetOrCreateAccessionWrapperCreator.convertToGetOrCreateAccessionWrapper( - Collections.singletonList(accessionWrapper))); + GetOrCreateAccessionWrapperCreator.convertToGetOrCreateAccessionWrapper( + Collections.singletonList(accessionWrapper))); resumingWriter.close(); } @@ -251,7 +249,7 @@ public void resumeWritingWithSameContigs() throws IOException { } @Test - @Ignore("A duplicated variant (two input variants that get the same accession) will appear twice in the VCF " + @Disabled("A duplicated variant (two input variants that get the same accession) will appear twice in the VCF " + "report if they are processed in different batches. There is no easy solution for this, as we would " + "need to cache previous batches. It is even worse if the job fails between the affected batches, as we " + "would need to parse the temporary VCF output or keep some variants in the job repository.") @@ -282,19 +280,19 @@ private void assertContigReplacement(String originalContig, String accessionedCo Variant variant = buildMockVariant(originalContig, accessionedContig, START_1, REFERENCE, ALTERNATE); SubmittedVariant submittedVariant = new SubmittedVariant("accession", TAXONOMY, "project", accessionedContig, - START_1, "", ALTERNATE, CLUSTERED_VARIANT, - SUPPORTED_BY_EVIDENCE, MATCHES_ASSEMBLY, ALLELES_MATCH, - VALIDATED, null); + START_1, "", ALTERNATE, CLUSTERED_VARIANT, + SUPPORTED_BY_EVIDENCE, MATCHES_ASSEMBLY, ALLELES_MATCH, + VALIDATED, null); AccessionWrapper accessionWrapper = new AccessionWrapper<>(ACCESSION, "hash-1", - submittedVariant); + submittedVariant); AccessionReportWriter accessionReportWriter = new AccessionReportWriter(output, fastaSequenceReader, - contigMapping, requestedReplacement); + contigMapping, requestedReplacement); accessionReportWriter.open(new ExecutionContext()); accessionReportWriter.write(Collections.singletonList(variant), - GetOrCreateAccessionWrapperCreator.convertToGetOrCreateAccessionWrapper( - Collections.singletonList(accessionWrapper))); + GetOrCreateAccessionWrapperCreator.convertToGetOrCreateAccessionWrapper( + Collections.singletonList(accessionWrapper))); assertEquals(replacementContig, getFirstVariantLine(variantsOutput).split("\t")[CHROMOSOME_COLUMN_VCF]); } @@ -304,11 +302,11 @@ public void writeChromosomeReplacedFromContig() throws IOException { assertContigReplacement(CHROMOSOME_1, CONTIG_1, ContigNaming.SEQUENCE_NAME, CHROMOSOME_1); } - @Test(expected = IllegalArgumentException.class) + @Test public void writeContigWithoutEquivalent() throws IOException { String contigMissingInAssemblyReport = "contig_missing_in_assembly_report"; - assertContigReplacement(contigMissingInAssemblyReport, contigMissingInAssemblyReport, ContigNaming.SEQUENCE_NAME, - contigMissingInAssemblyReport); + assertThrows(IllegalArgumentException.class, () -> assertContigReplacement(contigMissingInAssemblyReport, contigMissingInAssemblyReport, ContigNaming.SEQUENCE_NAME, + contigMissingInAssemblyReport)); } @Test @@ -324,7 +322,7 @@ public void writeContigWithoutIdenticalReplacement() throws IOException { /** * This test checks that the context base is added from the FASTA, and it doesn't matter which contig naming the * FASTA uses: it's independent of the accessioned contig naming (GenBank) and the VCF report contig naming. - * + *

* Note how in the fasta there's no entry for GENBANK_4 nor SEQUENCE_NAME_4, only for REFSEQ_4 */ @Test diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/batch/io/AccessionWriterTest.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/batch/io/AccessionWriterTest.java index 10fceca99..ab278fe38 100644 --- a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/batch/io/AccessionWriterTest.java +++ b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/batch/io/AccessionWriterTest.java @@ -16,15 +16,13 @@ package uk.ac.ebi.eva.accession.pipeline.batch.io; import org.hamcrest.Matchers; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; -import org.junit.rules.TemporaryFolder; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mockito; import org.springframework.batch.core.JobExecution; +import org.springframework.batch.item.Chunk; import org.springframework.batch.item.ExecutionContext; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; @@ -33,7 +31,7 @@ import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.ampt2d.commons.accession.core.models.AccessionWrapper; import uk.ac.ebi.eva.accession.core.batch.io.FastaSequenceReader; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.SubmittedVariantAccessioningConfiguration; @@ -45,10 +43,13 @@ import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; import uk.ac.ebi.eva.accession.core.service.GetOrCreateAccessionWrapperCreator; import uk.ac.ebi.eva.accession.core.service.nonhuman.SubmittedVariantAccessioningService; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; +import uk.ac.ebi.eva.accession.core.utils.PipelineTemporaryFolderUtil; import uk.ac.ebi.eva.accession.pipeline.batch.processors.VariantConverter; import uk.ac.ebi.eva.accession.pipeline.configuration.InputParametersConfiguration; import uk.ac.ebi.eva.accession.pipeline.configuration.batch.listeners.ListenersConfiguration; import uk.ac.ebi.eva.accession.pipeline.metric.AccessioningMetric; +import uk.ac.ebi.eva.accession.pipeline.test.MongoTestConfiguration; import uk.ac.ebi.eva.commons.core.models.pipeline.Variant; import uk.ac.ebi.eva.commons.core.models.pipeline.VariantSourceEntry; import uk.ac.ebi.eva.metrics.metric.MetricCompute; @@ -68,18 +69,19 @@ import java.util.List; import java.util.stream.Collectors; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertThat; -import static org.junit.Assert.assertTrue; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import static uk.ac.ebi.eva.accession.pipeline.batch.processors.ContigToGenbankReplacerProcessor.ORIGINAL_CHROMOSOME; -@RunWith(SpringRunner.class) +@ExtendWith(SpringExtension.class) @EnableAutoConfiguration @ContextConfiguration(classes = {SubmittedVariantAccessioningConfiguration.class, ListenersConfiguration.class, - InputParametersConfiguration.class}) + InputParametersConfiguration.class, MongoTestConfiguration.class}) @TestPropertySource("classpath:accession-pipeline-test.properties") -public class AccessionWriterTest { +public class AccessionWriterTest extends MongoTestContainerHelper { private static final int TAXONOMY = 3880; @@ -139,11 +141,7 @@ public class AccessionWriterTest { @MockBean private JobExecution jobExecution; - @Rule - public TemporaryFolder temporaryFolderRule = new TemporaryFolder(); - - @Rule - public ExpectedException thrown = ExpectedException.none(); + public PipelineTemporaryFolderUtil temporaryFolderUtil = new PipelineTemporaryFolderUtil(); private AccessionWriter accessionWriter; @@ -153,18 +151,18 @@ public class AccessionWriterTest { private VariantConverter variantConverter; - @Before + @BeforeEach public void setUp() throws Exception { contigMapping = new ContigMapping(Arrays.asList( new ContigSynonyms(CHROMOSOME_1, "assembled-molecule", "1", CONTIG_1, "refseq_1", "chr1", true), new ContigSynonyms(CHROMOSOME_2, "assembled-molecule", "2", CONTIG_2, "refseq_2", "chr2", true))); - output = temporaryFolderRule.newFile(); + output = temporaryFolderUtil.newFile(); variantOutput = new File(output.getAbsolutePath() + AccessionReportWriter.VARIANTS_FILE_SUFFIX); Path fastaPath = Paths.get(AccessionReportWriterTest.class.getResource("/input-files/fasta/mock.fa").toURI()); AccessionReportWriter accessionReportWriter = new AccessionReportWriter(output, - new FastaSequenceReader(fastaPath), - contigMapping, - ContigNaming.SEQUENCE_NAME); + new FastaSequenceReader(fastaPath), + contigMapping, + ContigNaming.SEQUENCE_NAME); variantConverter = new VariantConverter("assembly", TAXONOMY, "project"); accessionWriter = new AccessionWriter(service, accessionReportWriter, variantConverter, metricCompute); accessionReportWriter.open(new ExecutionContext()); @@ -173,8 +171,8 @@ public void setUp() throws Exception { accessionWriter.setJobExecution(jobExecution); } - @After - public void tearDown(){ + @AfterEach + public void tearDown() { metricCompute.clearCount(); } @@ -183,7 +181,7 @@ public void tearDown(){ public void saveSingleAccession() throws Exception { Variant variant = buildMockVariant("contig", START_1); - accessionWriter.write(Collections.singletonList(variant)); + accessionWriter.write(Chunk.of(variant)); ISubmittedVariant submittedVariant = variantConverter.convert(variant); List> accessions = service @@ -204,7 +202,7 @@ public void saveTwoAccession() throws Exception { Variant secondVariant = buildMockVariant("contig", START_2); List variants = Arrays.asList(firstVariant, secondVariant); - accessionWriter.write(variants); + accessionWriter.write(new Chunk<>(variants)); List> accessions = service.get(convert(variants)); assertEquals(2, accessions.size()); @@ -233,7 +231,7 @@ public void variantInsertionCheckOrder() throws Exception { Variant secondVariant = buildMockVariant(CONTIG_1, START_1, "", "A"); List variants = Arrays.asList(firstVariant, secondVariant); - accessionWriter.write(variants); + accessionWriter.write(new Chunk<>(variants)); List> accessions = service.get(convert(variants)); assertEquals(2, accessions.size()); @@ -241,7 +239,7 @@ public void variantInsertionCheckOrder() throws Exception { int firstVariantLineNumber = getVariantLineNumberByPosition(variantOutput, CHROMOSOME_1 + "\t" + START_1); //secondVariant position is START_1 - 1 because it is an insertion and the context base is added int secondVariantLineNumber = getVariantLineNumberByPosition(variantOutput, - CHROMOSOME_1 + "\t" + (START_1 - 1)); + CHROMOSOME_1 + "\t" + (START_1 - 1)); assertTrue(firstVariantLineNumber > secondVariantLineNumber); } @@ -263,7 +261,7 @@ private static int getVariantLineNumberByPosition(File output, String position) public void saveSameAccessionTwice() throws Exception { Variant variant = buildMockVariant("contig", START_1); - accessionWriter.write(Arrays.asList(variant, variant)); + accessionWriter.write(Chunk.of(variant, variant)); ISubmittedVariant submittedVariant = variantConverter.convert(variant); List> accessions = service @@ -281,7 +279,7 @@ public void saveSameAccessionTwice() throws Exception { public void testSaveInitializesCreatedDate() throws Exception { Variant variant = buildMockVariant("contig", START_1); LocalDateTime beforeSave = LocalDateTime.now(); - accessionWriter.write(Collections.singletonList(variant)); + accessionWriter.write(Chunk.of(variant)); LocalDateTime afterSave = LocalDateTime.now(); List> accessions = service @@ -297,7 +295,7 @@ public void testSaveInitializesCreatedDate() throws Exception { public void createAccessionAndItAppearsInTheReportVcf() throws Exception { Variant variant = buildMockVariant("contig", START_1); - accessionWriter.write(Arrays.asList(variant, variant)); + accessionWriter.write(Chunk.of(variant, variant)); List> accessions = service .get(Collections.singletonList(variantConverter.convert(variant))); @@ -305,36 +303,33 @@ public void createAccessionAndItAppearsInTheReportVcf() throws Exception { String vcfLine = AccessionReportWriterTest.getFirstVariantLine(variantOutput); assertEquals(vcfLine.split("\t")[ACCESSION_COLUMN], - ACCESSION_PREFIX + accessions.iterator().next().getAccession()); + ACCESSION_PREFIX + accessions.iterator().next().getAccession()); } @Test public void shouldThrowIfSomeVariantsWereNotAccessioned() { SubmittedVariant variant = new SubmittedVariant("assembly", TAXONOMY, "project", "contig", START_1, - REFERENCE_ALLELE, ALTERNATE_ALLELE, CLUSTERED_VARIANT, false, - MATCHES_ASSEMBLY, ALLELES_MATCH, VALIDATED, null); + REFERENCE_ALLELE, ALTERNATE_ALLELE, CLUSTERED_VARIANT, false, + MATCHES_ASSEMBLY, ALLELES_MATCH, VALIDATED, null); - thrown.expect(IllegalStateException.class); - accessionWriter.checkCountsMatch(Collections.singletonList(variant), new ArrayList<>()); + assertThrows(IllegalStateException.class, () -> accessionWriter.checkCountsMatch(Collections.singletonList(variant), new ArrayList<>())); } @Test public void shouldThrowIfSomeVariantsWereNotAccessionedInAChunkWithRepeatedVariants() { SubmittedVariant firstVariant = new SubmittedVariant("assembly", TAXONOMY, "project", "contig", START_1, - REFERENCE, ALTERNATE, CLUSTERED_VARIANT, false, - MATCHES_ASSEMBLY, ALLELES_MATCH, VALIDATED, null); + REFERENCE, ALTERNATE, CLUSTERED_VARIANT, false, + MATCHES_ASSEMBLY, ALLELES_MATCH, VALIDATED, null); SubmittedVariant secondVariant = new SubmittedVariant("assembly", TAXONOMY, "project", "contig", START_2, - REFERENCE, ALTERNATE, CLUSTERED_VARIANT, false, - MATCHES_ASSEMBLY, ALLELES_MATCH, VALIDATED, null); + REFERENCE, ALTERNATE, CLUSTERED_VARIANT, false, + MATCHES_ASSEMBLY, ALLELES_MATCH, VALIDATED, null); List variants = Arrays.asList(firstVariant, secondVariant, firstVariant, secondVariant); ArrayList> accessions = new ArrayList<>(); accessions.add(new AccessionWrapper<>(EXPECTED_ACCESSION, "hashedMessage", secondVariant)); - thrown.expect(IllegalStateException.class); - accessionWriter.checkCountsMatch(variants, - GetOrCreateAccessionWrapperCreator.convertToGetOrCreateAccessionWrapper( - accessions)); + assertThrows(IllegalStateException.class, () -> accessionWriter.checkCountsMatch(variants, + GetOrCreateAccessionWrapperCreator.convertToGetOrCreateAccessionWrapper(accessions))); } @Test @@ -343,26 +338,26 @@ public void shouldNotThrowIfVariantsDifferOnlyByClusteredVariantAccession() { SubmittedVariant variantWithRs = getSubmittedVariantWithClusteredVariant(NONNULL_CLUSTERED_VARIANT); List variants = Arrays.asList(variant, variant); AccessionWrapper accession = new AccessionWrapper<>(EXPECTED_ACCESSION, - "hashedMessage", - variantWithRs); + "hashedMessage", + variantWithRs); List> accessions = Collections.singletonList(accession); accessionWriter.checkCountsMatch(variants, - GetOrCreateAccessionWrapperCreator.convertToGetOrCreateAccessionWrapper( - accessions)); + GetOrCreateAccessionWrapperCreator.convertToGetOrCreateAccessionWrapper( + accessions)); } @Test public void shouldSortReport() throws Exception { // given List variants = Arrays.asList(buildMockVariant(CONTIG_2, CHROMOSOME_2, START_1), - buildMockVariant(CONTIG_1, CHROMOSOME_1, START_2), - buildMockVariant(CONTIG_3, CHROMOSOME_3, START_1), - buildMockVariant(CONTIG_1, CHROMOSOME_1, START_1), - buildMockVariant(CONTIG_2, CHROMOSOME_2, START_2)); + buildMockVariant(CONTIG_1, CHROMOSOME_1, START_2), + buildMockVariant(CONTIG_3, CHROMOSOME_3, START_1), + buildMockVariant(CONTIG_1, CHROMOSOME_1, START_1), + buildMockVariant(CONTIG_2, CHROMOSOME_2, START_2)); // when - accessionWriter.write(variants); + accessionWriter.write(new Chunk<>(variants)); // then BufferedReader fileInputStream = new BufferedReader(new InputStreamReader(new FileInputStream(variantOutput))); @@ -383,8 +378,8 @@ public void shouldSortReport() throws Exception { private SubmittedVariant getSubmittedVariantWithClusteredVariant(Long clusteredVariant) { return new SubmittedVariant("assembly", TAXONOMY, "project", "contig", START_1, - REFERENCE, ALTERNATE, clusteredVariant, false, - MATCHES_ASSEMBLY, ALLELES_MATCH, VALIDATED, null); + REFERENCE, ALTERNATE, clusteredVariant, false, + MATCHES_ASSEMBLY, ALLELES_MATCH, VALIDATED, null); } private Variant buildMockVariant(String contig, int start) { diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/batch/policies/InvalidVariantSkipPolicyTest.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/batch/policies/InvalidVariantSkipPolicyTest.java index efb92f988..b9dded92f 100644 --- a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/batch/policies/InvalidVariantSkipPolicyTest.java +++ b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/batch/policies/InvalidVariantSkipPolicyTest.java @@ -17,8 +17,8 @@ */ package uk.ac.ebi.eva.accession.pipeline.batch.policies; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import org.mockito.Mock; import org.springframework.batch.item.file.FlatFileParseException; @@ -26,8 +26,8 @@ import uk.ac.ebi.eva.commons.core.models.factories.exception.NonVariantException; import uk.ac.ebi.eva.commons.core.models.pipeline.Variant; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; public class InvalidVariantSkipPolicyTest { @@ -36,7 +36,7 @@ public class InvalidVariantSkipPolicyTest { private InvalidVariantSkipPolicy invalidVariantSkipPolicy; - @Before + @BeforeEach public void setUp() throws Exception { invalidVariantSkipPolicy = new InvalidVariantSkipPolicy(); } diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/batch/processors/ContigToGenbankReplacerProcessorTest.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/batch/processors/ContigToGenbankReplacerProcessorTest.java index 20b185530..5b362d695 100644 --- a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/batch/processors/ContigToGenbankReplacerProcessorTest.java +++ b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/batch/processors/ContigToGenbankReplacerProcessorTest.java @@ -15,8 +15,8 @@ */ package uk.ac.ebi.eva.accession.pipeline.batch.processors; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import uk.ac.ebi.eva.accession.core.contig.ContigMapping; import uk.ac.ebi.eva.commons.core.models.IVariant; import uk.ac.ebi.eva.commons.core.models.pipeline.Variant; @@ -26,15 +26,15 @@ import java.util.Set; import java.util.stream.Collectors; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThrows; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; import static uk.ac.ebi.eva.accession.pipeline.batch.processors.ContigToGenbankReplacerProcessor.ORIGINAL_CHROMOSOME; public class ContigToGenbankReplacerProcessorTest { private ContigToGenbankReplacerProcessor processor; - @Before + @BeforeEach public void setUp() throws Exception { String fileString = ContigToGenbankReplacerProcessorTest.class.getResource( "/input-files/assembly-report/assembly_report.txt").toString(); diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/batch/processors/ExcludeStructuralVariantsProcessorTest.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/batch/processors/ExcludeStructuralVariantsProcessorTest.java index ea221fe01..4b91f4549 100644 --- a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/batch/processors/ExcludeStructuralVariantsProcessorTest.java +++ b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/batch/processors/ExcludeStructuralVariantsProcessorTest.java @@ -17,8 +17,8 @@ */ package uk.ac.ebi.eva.accession.pipeline.batch.processors; -import org.junit.BeforeClass; -import org.junit.Test; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; import uk.ac.ebi.eva.commons.core.models.IVariant; import uk.ac.ebi.eva.commons.core.models.pipeline.Variant; @@ -26,9 +26,9 @@ import java.util.Arrays; import java.util.Objects; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; public class ExcludeStructuralVariantsProcessorTest { @@ -57,7 +57,7 @@ public class ExcludeStructuralVariantsProcessorTest { private static ExcludeStructuralVariantsProcessor processor; - @BeforeClass + @BeforeAll public static void setUp() { processor = new ExcludeStructuralVariantsProcessor(); } diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/batch/processors/VariantConverterTest.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/batch/processors/VariantConverterTest.java index 616c74a7d..b61ccaae3 100644 --- a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/batch/processors/VariantConverterTest.java +++ b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/batch/processors/VariantConverterTest.java @@ -17,15 +17,16 @@ */ package uk.ac.ebi.eva.accession.pipeline.batch.processors; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import uk.ac.ebi.eva.accession.core.model.ISubmittedVariant; import uk.ac.ebi.eva.accession.core.model.SubmittedVariant; import uk.ac.ebi.eva.commons.core.models.pipeline.Variant; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; public class VariantConverterTest { @@ -55,59 +56,59 @@ public class VariantConverterTest { private VariantConverter processor; - @Before + @BeforeEach public void setUp() { processor = new VariantConverter(ASSEMBLY, TAXONOMY, PROJECT); } @Test - public void process() throws Exception { + public void process() { Variant variant = new Variant(CONTIG, START, 1001, REFERENCE_ALLELE, ALTERNATE_ALLELE); ISubmittedVariant processed = processor.convert(variant); ISubmittedVariant expected = new SubmittedVariant(ASSEMBLY, TAXONOMY, PROJECT, CONTIG, START, REFERENCE_ALLELE, - ALTERNATE_ALLELE, CLUSTERED_VARIANT, SUPPORTED_BY_EVIDENCE, - MATCHES_ASSEMBLY, ALLELES_MATCH, VALIDATED, null); + ALTERNATE_ALLELE, CLUSTERED_VARIANT, SUPPORTED_BY_EVIDENCE, + MATCHES_ASSEMBLY, ALLELES_MATCH, VALIDATED, null); assertEquals(expected, processed); assertNull(processed.getCreatedDate()); } - @Test(expected = IllegalArgumentException.class) + @Test public void shouldThrowExceptionAssemblyNull() { - SubmittedVariant submittedVariant = new SubmittedVariant(null, TAXONOMY, PROJECT, CONTIG, START, - REFERENCE_ALLELE, ALTERNATE_ALLELE, CLUSTERED_VARIANT, - SUPPORTED_BY_EVIDENCE, MATCHES_ASSEMBLY, ALLELES_MATCH, - VALIDATED, null); + assertThrows(IllegalArgumentException.class, () -> new SubmittedVariant(null, TAXONOMY, PROJECT, CONTIG, START, + REFERENCE_ALLELE, ALTERNATE_ALLELE, CLUSTERED_VARIANT, + SUPPORTED_BY_EVIDENCE, MATCHES_ASSEMBLY, ALLELES_MATCH, + VALIDATED, null)); } - @Test(expected = IllegalArgumentException.class) + @Test public void shouldThrowExceptionProjectNull() { - SubmittedVariant submittedVariant = new SubmittedVariant(ASSEMBLY, TAXONOMY, null, CONTIG, START, - REFERENCE_ALLELE, ALTERNATE_ALLELE, CLUSTERED_VARIANT, - SUPPORTED_BY_EVIDENCE, MATCHES_ASSEMBLY, ALLELES_MATCH, - VALIDATED, null); + assertThrows(IllegalArgumentException.class, () -> new SubmittedVariant(ASSEMBLY, TAXONOMY, null, CONTIG, START, + REFERENCE_ALLELE, ALTERNATE_ALLELE, CLUSTERED_VARIANT, + SUPPORTED_BY_EVIDENCE, MATCHES_ASSEMBLY, ALLELES_MATCH, + VALIDATED, null)); } - @Test(expected = IllegalArgumentException.class) + @Test public void shouldThrowExceptionContigNull() { - SubmittedVariant submittedVariant = new SubmittedVariant(ASSEMBLY, TAXONOMY, PROJECT, null, START, - REFERENCE_ALLELE, ALTERNATE_ALLELE, CLUSTERED_VARIANT, - SUPPORTED_BY_EVIDENCE, MATCHES_ASSEMBLY, ALLELES_MATCH, - VALIDATED, null); + assertThrows(IllegalArgumentException.class, () -> new SubmittedVariant(ASSEMBLY, TAXONOMY, PROJECT, null, START, + REFERENCE_ALLELE, ALTERNATE_ALLELE, CLUSTERED_VARIANT, + SUPPORTED_BY_EVIDENCE, MATCHES_ASSEMBLY, ALLELES_MATCH, + VALIDATED, null)); } - @Test(expected = IllegalArgumentException.class) + @Test public void shouldThrowExceptionReferenceNull() { - SubmittedVariant submittedVariant = new SubmittedVariant(ASSEMBLY, TAXONOMY, PROJECT, CONTIG, START, null, - ALTERNATE_ALLELE, CLUSTERED_VARIANT, - SUPPORTED_BY_EVIDENCE, MATCHES_ASSEMBLY, ALLELES_MATCH, - VALIDATED, null); + assertThrows(IllegalArgumentException.class, () -> new SubmittedVariant(ASSEMBLY, TAXONOMY, PROJECT, CONTIG, START, null, + ALTERNATE_ALLELE, CLUSTERED_VARIANT, + SUPPORTED_BY_EVIDENCE, MATCHES_ASSEMBLY, ALLELES_MATCH, + VALIDATED, null)); } - @Test(expected = IllegalArgumentException.class) + @Test public void shouldThrowExceptionAlternateNull() { - SubmittedVariant submittedVariant = new SubmittedVariant(ASSEMBLY, TAXONOMY, PROJECT, CONTIG, START, - REFERENCE_ALLELE, null, CLUSTERED_VARIANT, - SUPPORTED_BY_EVIDENCE, MATCHES_ASSEMBLY, ALLELES_MATCH, - VALIDATED, null); + assertThrows(IllegalArgumentException.class, () -> new SubmittedVariant(ASSEMBLY, TAXONOMY, PROJECT, CONTIG, START, + REFERENCE_ALLELE, null, CLUSTERED_VARIANT, + SUPPORTED_BY_EVIDENCE, MATCHES_ASSEMBLY, ALLELES_MATCH, + VALIDATED, null)); } } diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/batch/tasklets/buildReport/BuildReportTaskletTest.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/batch/tasklets/buildReport/BuildReportTaskletTest.java index 279b47038..56189db60 100644 --- a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/batch/tasklets/buildReport/BuildReportTaskletTest.java +++ b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/batch/tasklets/buildReport/BuildReportTaskletTest.java @@ -15,11 +15,9 @@ */ package uk.ac.ebi.eva.accession.pipeline.batch.tasklets.buildReport; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; - +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import uk.ac.ebi.eva.accession.core.utils.PipelineTemporaryFolderUtil; import uk.ac.ebi.eva.accession.pipeline.batch.io.AccessionReportWriter; import java.io.BufferedReader; @@ -32,10 +30,10 @@ import java.nio.file.Files; import java.util.Arrays; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; public class BuildReportTaskletTest { @@ -55,8 +53,7 @@ public class BuildReportTaskletTest { private static final int CHROMOSOME_COLUMN_VCF = 0; - @Rule - public TemporaryFolder temporaryFolderRule = new TemporaryFolder(); + public PipelineTemporaryFolderUtil temporaryFolderUtil = new PipelineTemporaryFolderUtil(); private File output; @@ -64,9 +61,9 @@ public class BuildReportTaskletTest { private File contigsOutput; - @Before + @BeforeEach public void setUp() throws Exception { - output = temporaryFolderRule.newFile(); + output = temporaryFolderUtil.newFile(); variantsOutput = new File(output.getAbsolutePath() + AccessionReportWriter.VARIANTS_FILE_SUFFIX); contigsOutput = new File(output.getAbsolutePath() + AccessionReportWriter.CONTIGS_FILE_SUFFIX); } @@ -97,7 +94,7 @@ private void assertHeaderIsNotWrittenTwice(File output) throws IOException { String variantLine = line; do { - assertFalse("VCF report has header lines after variant lines", variantLine.startsWith("#")); + assertFalse(variantLine.startsWith("#"), "VCF report has header lines after variant lines"); variantLine = fileInputStream.readLine(); } while (variantLine != null); } @@ -131,8 +128,8 @@ private void writeSingleVariantInContigAndVariantFiles(String originalChromosome FileWriter variantsWriter = new FileWriter(variantsOutput); variantsWriter.write(String.join("\t", - Arrays.asList(originalChromosome, START_1.toString(), ACCESSION.toString(), - REFERENCE, ALTERNATE, MISSING, MISSING, MISSING))); + Arrays.asList(originalChromosome, START_1.toString(), ACCESSION.toString(), + REFERENCE, ALTERNATE, MISSING, MISSING, MISSING))); variantsWriter.close(); } diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/batch/tasklets/reportCheck/AccessionWrapperComparatorTest.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/batch/tasklets/reportCheck/AccessionWrapperComparatorTest.java index 923c318ca..68b48ce41 100644 --- a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/batch/tasklets/reportCheck/AccessionWrapperComparatorTest.java +++ b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/batch/tasklets/reportCheck/AccessionWrapperComparatorTest.java @@ -15,12 +15,9 @@ */ package uk.ac.ebi.eva.accession.pipeline.batch.tasklets.reportCheck; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import uk.ac.ebi.ampt2d.commons.accession.core.models.AccessionWrapper; - import uk.ac.ebi.eva.accession.core.model.ISubmittedVariant; import uk.ac.ebi.eva.accession.core.model.SubmittedVariant; import uk.ac.ebi.eva.commons.core.models.pipeline.Variant; @@ -29,7 +26,8 @@ import java.util.List; import java.util.Map; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; public class AccessionWrapperComparatorTest { @@ -51,12 +49,9 @@ public class AccessionWrapperComparatorTest { private AccessionWrapperComparator accessionWrapperComparator; - @Rule - public ExpectedException thrown = ExpectedException.none(); - private Map contigMapping; - @Before + @BeforeEach public void setUp() throws Exception { List variants = Arrays.asList(buildMockVariant(CONTIG_A), buildMockVariant(CONTIG_B), @@ -162,8 +157,8 @@ public void checkUnexpectedContigRaisesException() { buildMockVariant(CONTIG_A)); AccessionWrapperComparator comparator = new AccessionWrapperComparator(variants); - thrown.expect(IllegalStateException.class); - comparator.compare(buildMockAccessionWrapper(CONTIG_11, 100), - buildMockAccessionWrapper(CONTIG_2, 100)); + assertThrows(IllegalStateException.class, () -> + comparator.compare(buildMockAccessionWrapper(CONTIG_11, 100), + buildMockAccessionWrapper(CONTIG_2, 100))); } } diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/batch/tasklets/reportCheck/ReportCheckTaskletTest.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/batch/tasklets/reportCheck/ReportCheckTaskletTest.java index 61dc7f790..a64cf047c 100644 --- a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/batch/tasklets/reportCheck/ReportCheckTaskletTest.java +++ b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/batch/tasklets/reportCheck/ReportCheckTaskletTest.java @@ -15,7 +15,7 @@ */ package uk.ac.ebi.eva.accession.pipeline.batch.tasklets.reportCheck; -import org.junit.Test; +import org.junit.jupiter.api.Test; import org.springframework.batch.core.ExitStatus; import org.springframework.batch.core.JobExecution; import org.springframework.batch.core.StepContribution; @@ -26,10 +26,9 @@ import org.springframework.batch.item.NonTransientResourceException; import org.springframework.batch.item.ParseException; import org.springframework.batch.item.UnexpectedInputException; - +import uk.ac.ebi.eva.accession.core.batch.io.AccessionedVcfLineMapper; import uk.ac.ebi.eva.accession.core.contig.ContigMapping; import uk.ac.ebi.eva.accession.core.contig.ContigSynonyms; -import uk.ac.ebi.eva.accession.core.batch.io.AccessionedVcfLineMapper; import uk.ac.ebi.eva.commons.batch.io.AggregatedVcfReader; import uk.ac.ebi.eva.commons.batch.io.UnwindingItemStreamReader; import uk.ac.ebi.eva.commons.batch.io.VcfReader; @@ -43,7 +42,7 @@ import java.util.Collections; import java.util.Iterator; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.QC_SUBSNP_ACCESSION_STEP; public class ReportCheckTaskletTest { diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSeveralIntervalsOfSubsnpAccessionsJobConfigurationTest.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSeveralIntervalsOfSubsnpAccessionsJobConfigurationTest.java index 6ae36d3c6..b65a1f30f 100644 --- a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSeveralIntervalsOfSubsnpAccessionsJobConfigurationTest.java +++ b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSeveralIntervalsOfSubsnpAccessionsJobConfigurationTest.java @@ -16,10 +16,10 @@ package uk.ac.ebi.eva.accession.pipeline.configuration.batch.jobs; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.batch.core.BatchStatus; import org.springframework.batch.core.JobExecution; import org.springframework.batch.core.StepExecution; @@ -32,16 +32,19 @@ import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import org.springframework.test.web.client.ExpectedCount; import org.springframework.test.web.client.MockRestServiceServer; import org.springframework.web.client.RestTemplate; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.SubmittedVariantAccessioningConfiguration; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; import uk.ac.ebi.eva.accession.core.repository.nonhuman.eva.SubmittedVariantAccessioningRepository; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.pipeline.batch.io.AccessionReportWriter; import uk.ac.ebi.eva.accession.pipeline.parameters.InputParameters; +import uk.ac.ebi.eva.accession.pipeline.test.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.accession.pipeline.test.BatchTestConfiguration; +import uk.ac.ebi.eva.accession.pipeline.test.MongoTestConfiguration; import uk.ac.ebi.eva.commons.core.utils.FileUtils; import uk.ac.ebi.eva.metrics.count.CountServiceParameters; @@ -58,7 +61,7 @@ import java.util.List; import java.util.TreeSet; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.springframework.test.web.client.match.MockRestRequestMatchers.method; import static org.springframework.test.web.client.match.MockRestRequestMatchers.requestTo; import static org.springframework.test.web.client.response.MockRestResponseCreators.withStatus; @@ -67,10 +70,11 @@ import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.SUBSNP_ACCESSION_STEP; import static uk.ac.ebi.eva.accession.pipeline.test.BatchTestConfiguration.JOB_LAUNCHER_SUBSNP_ACCESSION_JOB; -@RunWith(SpringRunner.class) -@ContextConfiguration(classes = {BatchTestConfiguration.class, SubmittedVariantAccessioningConfiguration.class}) +@ExtendWith(SpringExtension.class) +@ContextConfiguration(classes = {BatchTestConfiguration.class, SubmittedVariantAccessioningConfiguration.class, + MongoTestConfiguration.class, BatchJobRepositoryTestConfiguration.class}) @TestPropertySource("classpath:accession-pipeline-interval-test.properties") -public class CreateSeveralIntervalsOfSubsnpAccessionsJobConfigurationTest { +public class CreateSeveralIntervalsOfSubsnpAccessionsJobConfigurationTest extends MongoTestContainerHelper { private static final int EXPECTED_VARIANTS = 22; @@ -98,7 +102,7 @@ public class CreateSeveralIntervalsOfSubsnpAccessionsJobConfigurationTest { private MockRestServiceServer mockServer; - @Before + @BeforeEach public void setUp() throws Exception { this.cleanSlate(); mongoTemplate.dropCollection(SubmittedVariantEntity.class); @@ -108,7 +112,7 @@ public void setUp() throws Exception { .andRespond(withStatus(HttpStatus.OK)); } - @After + @AfterEach public void tearDown() throws Exception { this.cleanSlate(); mongoTemplate.dropCollection(SubmittedVariantEntity.class); diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSubsnpAccessionsJobConfigurationTest.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSubsnpAccessionsJobConfigurationTest.java index 02b005aab..1673ca54a 100644 --- a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSubsnpAccessionsJobConfigurationTest.java +++ b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSubsnpAccessionsJobConfigurationTest.java @@ -16,10 +16,10 @@ package uk.ac.ebi.eva.accession.pipeline.configuration.batch.jobs; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.batch.core.BatchStatus; import org.springframework.batch.core.JobExecution; import org.springframework.batch.core.StepExecution; @@ -32,16 +32,19 @@ import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import org.springframework.test.web.client.ExpectedCount; import org.springframework.test.web.client.MockRestServiceServer; import org.springframework.web.client.RestTemplate; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.SubmittedVariantAccessioningConfiguration; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; import uk.ac.ebi.eva.accession.core.repository.nonhuman.eva.SubmittedVariantAccessioningRepository; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.pipeline.batch.io.AccessionReportWriter; import uk.ac.ebi.eva.accession.pipeline.parameters.InputParameters; +import uk.ac.ebi.eva.accession.pipeline.test.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.accession.pipeline.test.BatchTestConfiguration; +import uk.ac.ebi.eva.accession.pipeline.test.MongoTestConfiguration; import uk.ac.ebi.eva.commons.core.utils.FileUtils; import uk.ac.ebi.eva.metrics.count.CountServiceParameters; @@ -56,8 +59,8 @@ import java.util.Collection; import java.util.Iterator; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.springframework.test.web.client.match.MockRestRequestMatchers.method; import static org.springframework.test.web.client.match.MockRestRequestMatchers.requestTo; import static org.springframework.test.web.client.response.MockRestResponseCreators.withStatus; @@ -66,10 +69,11 @@ import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.SUBSNP_ACCESSION_STEP; import static uk.ac.ebi.eva.accession.pipeline.test.BatchTestConfiguration.JOB_LAUNCHER_SUBSNP_ACCESSION_JOB; -@RunWith(SpringRunner.class) -@ContextConfiguration(classes = {BatchTestConfiguration.class, SubmittedVariantAccessioningConfiguration.class}) +@ExtendWith(SpringExtension.class) +@ContextConfiguration(classes = {BatchTestConfiguration.class, SubmittedVariantAccessioningConfiguration.class, + MongoTestConfiguration.class, BatchJobRepositoryTestConfiguration.class}) @TestPropertySource("classpath:accession-pipeline-test.properties") -public class CreateSubsnpAccessionsJobConfigurationTest { +public class CreateSubsnpAccessionsJobConfigurationTest extends MongoTestContainerHelper { private static final int EXPECTED_VARIANTS = 22; @@ -97,7 +101,7 @@ public class CreateSubsnpAccessionsJobConfigurationTest { private MockRestServiceServer mockServer; - @Before + @BeforeEach public void setUp() throws Exception { this.cleanSlate(); mongoTemplate.dropCollection(SubmittedVariantEntity.class); @@ -107,7 +111,7 @@ public void setUp() throws Exception { .andRespond(withStatus(HttpStatus.OK)); } - @After + @AfterEach public void tearDown() throws Exception { this.cleanSlate(); mongoTemplate.dropCollection(SubmittedVariantEntity.class); diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSubsnpAccessionsRecoverStateTest.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSubsnpAccessionsRecoverStateTest.java index 59f1ab36a..f1e8b3ad1 100644 --- a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSubsnpAccessionsRecoverStateTest.java +++ b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSubsnpAccessionsRecoverStateTest.java @@ -16,26 +16,22 @@ package uk.ac.ebi.eva.accession.pipeline.configuration.batch.jobs; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.batch.core.BatchStatus; import org.springframework.batch.core.JobExecution; import org.springframework.batch.core.StepExecution; import org.springframework.batch.test.JobLauncherTestUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.context.ApplicationContext; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.http.HttpMethod; import org.springframework.http.HttpStatus; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import org.springframework.test.web.client.ExpectedCount; import org.springframework.test.web.client.MockRestServiceServer; import org.springframework.web.client.RestTemplate; @@ -45,10 +41,12 @@ import uk.ac.ebi.eva.accession.core.model.SubmittedVariant; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; import uk.ac.ebi.eva.accession.core.repository.nonhuman.eva.SubmittedVariantAccessioningRepository; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.pipeline.batch.io.AccessionReportWriter; import uk.ac.ebi.eva.accession.pipeline.parameters.InputParameters; +import uk.ac.ebi.eva.accession.pipeline.test.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.accession.pipeline.test.BatchTestConfiguration; -import uk.ac.ebi.eva.accession.pipeline.test.FixSpringMongoDbRule; +import uk.ac.ebi.eva.accession.pipeline.test.MongoTestConfiguration; import uk.ac.ebi.eva.commons.core.utils.FileUtils; import uk.ac.ebi.eva.metrics.count.CountServiceParameters; @@ -62,8 +60,8 @@ import java.util.Iterator; import java.util.List; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.springframework.test.web.client.match.MockRestRequestMatchers.method; import static org.springframework.test.web.client.match.MockRestRequestMatchers.requestTo; import static org.springframework.test.web.client.response.MockRestResponseCreators.withStatus; @@ -72,11 +70,11 @@ import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.SUBSNP_ACCESSION_STEP; import static uk.ac.ebi.eva.accession.pipeline.test.BatchTestConfiguration.JOB_LAUNCHER_SUBSNP_ACCESSION_JOB; -@RunWith(SpringRunner.class) -@ContextConfiguration(classes = {BatchTestConfiguration.class, SubmittedVariantAccessioningConfiguration.class}) +@ExtendWith(SpringExtension.class) +@ContextConfiguration(classes = {BatchTestConfiguration.class, SubmittedVariantAccessioningConfiguration.class, + MongoTestConfiguration.class, BatchJobRepositoryTestConfiguration.class}) @TestPropertySource("classpath:accession-pipeline-recover-state-test.properties") -public class CreateSubsnpAccessionsRecoverStateTest { - private static final String TEST_DB = "test-db"; +public class CreateSubsnpAccessionsRecoverStateTest extends MongoTestContainerHelper { @Autowired private SubmittedVariantAccessioningRepository mongoRepository; @@ -90,14 +88,6 @@ public class CreateSubsnpAccessionsRecoverStateTest { @Autowired private MongoTemplate mongoTemplate; - //needed for @UsingDataSet - @Autowired - private ApplicationContext applicationContext; - - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName(TEST_DB).build()); - @Autowired @Qualifier(JOB_LAUNCHER_SUBSNP_ACCESSION_JOB) private JobLauncherTestUtils jobLauncherTestUtils; @@ -115,8 +105,9 @@ public class CreateSubsnpAccessionsRecoverStateTest { private MockRestServiceServer mockServer; - @Before + @BeforeEach public void setUp() throws Exception { + mongoTemplate.getDb().drop(); this.cleanSlate(); mockServer = MockRestServiceServer.createServer(restTemplate); mockServer.expect(ExpectedCount.manyTimes(), requestTo(new URI(countServiceParameters.getUrl() + URL_PATH_SAVE_COUNT))) @@ -124,10 +115,10 @@ public void setUp() throws Exception { .andRespond(withStatus(HttpStatus.OK)); } - @After + @AfterEach public void tearDown() throws Exception { + mongoTemplate.getDb().drop(); this.cleanSlate(); - mongoTemplate.dropCollection(SubmittedVariantEntity.class); } public void cleanSlate() throws Exception { diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSubsnpAccessionsRecoveringStateJobConfigurationTest.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSubsnpAccessionsRecoveringStateJobConfigurationTest.java index 09fdb091c..78ea91fb3 100644 --- a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSubsnpAccessionsRecoveringStateJobConfigurationTest.java +++ b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSubsnpAccessionsRecoveringStateJobConfigurationTest.java @@ -16,26 +16,22 @@ package uk.ac.ebi.eva.accession.pipeline.configuration.batch.jobs; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.batch.core.BatchStatus; import org.springframework.batch.core.JobExecution; import org.springframework.batch.core.StepExecution; import org.springframework.batch.test.JobLauncherTestUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.context.ApplicationContext; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.http.HttpMethod; import org.springframework.http.HttpStatus; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import org.springframework.test.web.client.ExpectedCount; import org.springframework.test.web.client.MockRestServiceServer; import org.springframework.web.client.RestTemplate; @@ -45,10 +41,12 @@ import uk.ac.ebi.eva.accession.core.model.SubmittedVariant; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; import uk.ac.ebi.eva.accession.core.repository.nonhuman.eva.SubmittedVariantAccessioningRepository; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.pipeline.batch.io.AccessionReportWriter; import uk.ac.ebi.eva.accession.pipeline.parameters.InputParameters; +import uk.ac.ebi.eva.accession.pipeline.test.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.accession.pipeline.test.BatchTestConfiguration; -import uk.ac.ebi.eva.accession.pipeline.test.FixSpringMongoDbRule; +import uk.ac.ebi.eva.accession.pipeline.test.MongoTestConfiguration; import uk.ac.ebi.eva.commons.core.utils.FileUtils; import uk.ac.ebi.eva.metrics.count.CountServiceParameters; @@ -60,7 +58,7 @@ import java.util.Collection; import java.util.Iterator; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.springframework.test.web.client.match.MockRestRequestMatchers.method; import static org.springframework.test.web.client.match.MockRestRequestMatchers.requestTo; import static org.springframework.test.web.client.response.MockRestResponseCreators.withStatus; @@ -69,17 +67,16 @@ import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.SUBSNP_ACCESSION_STEP; import static uk.ac.ebi.eva.accession.pipeline.test.BatchTestConfiguration.JOB_LAUNCHER_SUBSNP_ACCESSION_JOB; -@RunWith(SpringRunner.class) -@ContextConfiguration(classes = {BatchTestConfiguration.class, SubmittedVariantAccessioningConfiguration.class}) +@ExtendWith(SpringExtension.class) +@ContextConfiguration(classes = {BatchTestConfiguration.class, SubmittedVariantAccessioningConfiguration.class, + MongoTestConfiguration.class, BatchJobRepositoryTestConfiguration.class}) @TestPropertySource("classpath:accession-pipeline-recover-test.properties") -public class CreateSubsnpAccessionsRecoveringStateJobConfigurationTest { +public class CreateSubsnpAccessionsRecoveringStateJobConfigurationTest extends MongoTestContainerHelper { public static final long UNCOMMITTED_ACCESSION = 5000000000L; private static final int EXPECTED_VARIANTS_ACCESSIONED_FROM_VCF = 22; - private static final String TEST_DB = "test-db"; - @Autowired private CountServiceParameters countServiceParameters; @@ -103,20 +100,13 @@ public class CreateSubsnpAccessionsRecoveringStateJobConfigurationTest { @Autowired private MongoTemplate mongoTemplate; - //needed for @UsingDataSet - @Autowired - private ApplicationContext applicationContext; - - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName(TEST_DB).build()); - @Autowired @Qualifier(JOB_LAUNCHER_SUBSNP_ACCESSION_JOB) private JobLauncherTestUtils jobLauncherTestUtils; - @Before + @BeforeEach public void setUp() throws Exception { + mongoTemplate.getDb().drop(); this.cleanSlate(); mockServer = MockRestServiceServer.createServer(restTemplate); mockServer.expect(ExpectedCount.manyTimes(), requestTo(new URI(countServiceParameters.getUrl() + URL_PATH_SAVE_COUNT))) @@ -124,10 +114,10 @@ public void setUp() throws Exception { .andRespond(withStatus(HttpStatus.OK)); } - @After + @AfterEach public void tearDown() throws Exception { + mongoTemplate.getDb().drop(); this.cleanSlate(); - mongoTemplate.dropCollection(SubmittedVariantEntity.class); } public void cleanSlate() throws Exception { @@ -143,8 +133,8 @@ public void accessionJobShouldRecoverUncommittedAccessions() throws Exception { runJob(); - assertEquals("The uncommitted accession should be assigned to only 1 object", - 1, repository.findByAccession(UNCOMMITTED_ACCESSION).size()); + assertEquals(1, repository.findByAccession(UNCOMMITTED_ACCESSION).size(), + "The uncommitted accession should be assigned to only 1 object"); assertCountsInMongo(EXPECTED_VARIANTS_ACCESSIONED_FROM_VCF + 1); assertCountsInVcfReport(EXPECTED_VARIANTS_ACCESSIONED_FROM_VCF); diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/DuplicateSSAccQCJobConfigurationTest.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/DuplicateSSAccQCJobConfigurationTest.java index 0348ee6a3..8d40527a4 100644 --- a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/DuplicateSSAccQCJobConfigurationTest.java +++ b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/DuplicateSSAccQCJobConfigurationTest.java @@ -15,30 +15,25 @@ */ package uk.ac.ebi.eva.accession.pipeline.configuration.batch.jobs; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; -import com.mongodb.MongoClient; -import gherkin.deps.com.google.gson.Gson; -import gherkin.deps.com.google.gson.GsonBuilder; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.batch.core.BatchStatus; import org.springframework.batch.core.JobExecution; import org.springframework.batch.test.JobLauncherTestUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.context.ApplicationContext; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpSubmittedVariantEntity; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; +import uk.ac.ebi.eva.accession.pipeline.test.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.accession.pipeline.test.BatchTestConfiguration; -import uk.ac.ebi.eva.accession.pipeline.test.FixSpringMongoDbRule; +import uk.ac.ebi.eva.accession.pipeline.test.MongoTestConfiguration; import java.io.IOException; import java.nio.file.Files; @@ -47,48 +42,34 @@ import java.util.Set; import java.util.stream.Collectors; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.DUPLICATE_SS_ACC_QC_STEP; import static uk.ac.ebi.eva.accession.pipeline.test.BatchTestConfiguration.JOB_LAUNCHER_DUPLICATE_SS_ACC_QC_JOB; -@RunWith(SpringRunner.class) -@ContextConfiguration(classes = {BatchTestConfiguration.class}) +@ExtendWith(SpringExtension.class) +@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class, + BatchJobRepositoryTestConfiguration.class}) @TestPropertySource("classpath:duplicate-ss-acc-qc-test.properties") -public class DuplicateSSAccQCJobConfigurationTest { - - private static final String TEST_DB = "test-db"; +public class DuplicateSSAccQCJobConfigurationTest extends MongoTestContainerHelper { private static final String duplicateSsAccFile = "src/test/resources/duplicateSSAcc.csv"; @Autowired @Qualifier(JOB_LAUNCHER_DUPLICATE_SS_ACC_QC_JOB) private JobLauncherTestUtils jobLauncherTestUtils; - @Autowired - private MongoClient mongoClient; - @Autowired private MongoTemplate mongoTemplate; - //Required by nosql-unit - @Autowired - private ApplicationContext applicationContext; - - private final Gson gson = new GsonBuilder().create(); - - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName(TEST_DB).build()); - - @Before + @BeforeEach public void setUp() throws IOException { - this.mongoClient.dropDatabase(TEST_DB); + mongoTemplate.getDb().drop(); Files.deleteIfExists(Paths.get(duplicateSsAccFile)); } - @After + @AfterEach public void tearDown() throws IOException { - this.mongoClient.dropDatabase(TEST_DB); + mongoTemplate.getDb().drop(); Files.deleteIfExists(Paths.get(duplicateSsAccFile)); } diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/QCSubsnpAccessionsJobConfigurationTest.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/QCSubsnpAccessionsJobConfigurationTest.java index 295c16d0d..9bf30a7ba 100644 --- a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/QCSubsnpAccessionsJobConfigurationTest.java +++ b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/QCSubsnpAccessionsJobConfigurationTest.java @@ -16,8 +16,8 @@ package uk.ac.ebi.eva.accession.pipeline.configuration.batch.jobs; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.batch.core.BatchStatus; import org.springframework.batch.core.ExitStatus; import org.springframework.batch.core.JobExecution; @@ -28,20 +28,24 @@ import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.SubmittedVariantAccessioningConfiguration; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; +import uk.ac.ebi.eva.accession.pipeline.test.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.accession.pipeline.test.BatchTestConfiguration; +import uk.ac.ebi.eva.accession.pipeline.test.MongoTestConfiguration; import java.util.Collection; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.QC_SUBSNP_ACCESSION_STEP; import static uk.ac.ebi.eva.accession.pipeline.test.BatchTestConfiguration.JOB_LAUNCHER_QC_SUBSNP_ACCESSION_JOB; -@RunWith(SpringRunner.class) -@ContextConfiguration(classes = {BatchTestConfiguration.class, SubmittedVariantAccessioningConfiguration.class}) +@ExtendWith(SpringExtension.class) +@ContextConfiguration(classes = {BatchTestConfiguration.class, SubmittedVariantAccessioningConfiguration.class, + MongoTestConfiguration.class, BatchJobRepositoryTestConfiguration.class}) @TestPropertySource("classpath:qc-accession-pipeline-test.properties") -public class QCSubsnpAccessionsJobConfigurationTest { +public class QCSubsnpAccessionsJobConfigurationTest extends MongoTestContainerHelper { @Autowired @Qualifier(JOB_LAUNCHER_QC_SUBSNP_ACCESSION_JOB) private JobLauncherTestUtils jobLauncherTestUtils; diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/CreateSubsnpAccessionsStepConfigurationTest.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/CreateSubsnpAccessionsStepConfigurationTest.java index 19c0c97de..267a39dba 100644 --- a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/CreateSubsnpAccessionsStepConfigurationTest.java +++ b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/CreateSubsnpAccessionsStepConfigurationTest.java @@ -16,10 +16,10 @@ package uk.ac.ebi.eva.accession.pipeline.configuration.batch.steps; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mockito; import org.springframework.batch.core.BatchStatus; import org.springframework.batch.core.ExitStatus; @@ -33,15 +33,18 @@ import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.ampt2d.commons.accession.persistence.jpa.monotonic.service.ContiguousIdBlockService; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.SubmittedVariantAccessioningConfiguration; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; import uk.ac.ebi.eva.accession.core.repository.nonhuman.eva.SubmittedVariantAccessioningRepository; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.pipeline.batch.io.AccessionReportWriter; import uk.ac.ebi.eva.accession.pipeline.batch.io.AccessionWriter; import uk.ac.ebi.eva.accession.pipeline.parameters.InputParameters; +import uk.ac.ebi.eva.accession.pipeline.test.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.accession.pipeline.test.BatchTestConfiguration; +import uk.ac.ebi.eva.accession.pipeline.test.MongoTestConfiguration; import uk.ac.ebi.eva.commons.core.utils.FileUtils; import java.io.FileInputStream; @@ -49,15 +52,16 @@ import java.nio.file.Files; import java.nio.file.Paths; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.Mockito.when; import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.SUBSNP_ACCESSION_STEP; import static uk.ac.ebi.eva.accession.pipeline.test.BatchTestConfiguration.JOB_LAUNCHER_SUBSNP_ACCESSION_JOB; -@RunWith(SpringRunner.class) -@ContextConfiguration(classes = {BatchTestConfiguration.class, SubmittedVariantAccessioningConfiguration.class}) +@ExtendWith(SpringExtension.class) +@ContextConfiguration(classes = {BatchTestConfiguration.class, SubmittedVariantAccessioningConfiguration.class, + MongoTestConfiguration.class, BatchJobRepositoryTestConfiguration.class}) @TestPropertySource("classpath:accession-pipeline-test.properties") -public class CreateSubsnpAccessionsStepConfigurationTest { +public class CreateSubsnpAccessionsStepConfigurationTest extends MongoTestContainerHelper { private static final int EXPECTED_VARIANTS = 22; @@ -85,7 +89,7 @@ public class CreateSubsnpAccessionsStepConfigurationTest { @SpyBean private ContiguousIdBlockService contiguousIdBlockService; - @Before + @BeforeEach public void setUp() throws Exception { // if a new transaction is not created it will fail when(contiguousIdBlockService.getBlockParameters("test-pipeline-ss")) @@ -104,7 +108,7 @@ public void setUp() throws Exception { accessionWriter.setJobExecution(jobExecution); } - @After + @AfterEach public void tearDown() throws Exception { Files.deleteIfExists(Paths.get(inputParameters.getOutputVcf())); Files.deleteIfExists(Paths.get(inputParameters.getOutputVcf() + AccessionReportWriter.VARIANTS_FILE_SUFFIX)); diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/QCSubsnpAccessionsStepConfigurationTest.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/QCSubsnpAccessionsStepConfigurationTest.java index aca599cd9..1737d3d62 100644 --- a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/QCSubsnpAccessionsStepConfigurationTest.java +++ b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/QCSubsnpAccessionsStepConfigurationTest.java @@ -16,9 +16,9 @@ package uk.ac.ebi.eva.accession.pipeline.configuration.batch.steps; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mockito; import org.springframework.batch.core.BatchStatus; import org.springframework.batch.core.ExitStatus; @@ -30,19 +30,23 @@ import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.SubmittedVariantAccessioningConfiguration; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.pipeline.batch.io.AccessionWriter; +import uk.ac.ebi.eva.accession.pipeline.test.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.accession.pipeline.test.BatchTestConfiguration; +import uk.ac.ebi.eva.accession.pipeline.test.MongoTestConfiguration; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.QC_SUBSNP_ACCESSION_STEP; import static uk.ac.ebi.eva.accession.pipeline.test.BatchTestConfiguration.JOB_LAUNCHER_QC_SUBSNP_ACCESSION_JOB; -@RunWith(SpringRunner.class) -@ContextConfiguration(classes = {BatchTestConfiguration.class, SubmittedVariantAccessioningConfiguration.class}) +@ExtendWith(SpringExtension.class) +@ContextConfiguration(classes = {BatchTestConfiguration.class, SubmittedVariantAccessioningConfiguration.class, + MongoTestConfiguration.class, BatchJobRepositoryTestConfiguration.class}) @TestPropertySource("classpath:qc-accession-pipeline-test.properties") -public class QCSubsnpAccessionsStepConfigurationTest { +public class QCSubsnpAccessionsStepConfigurationTest extends MongoTestContainerHelper { @Autowired @Qualifier(JOB_LAUNCHER_QC_SUBSNP_ACCESSION_JOB) private JobLauncherTestUtils jobLauncherTestUtils; @@ -53,7 +57,7 @@ public class QCSubsnpAccessionsStepConfigurationTest { @MockBean private JobExecution jobExecution; - @Before + @BeforeEach public void setUp() throws Exception { Mockito.when(jobExecution.getJobId()).thenReturn(1L); accessionWriter.setJobExecution(jobExecution); diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/runner/EvaAccessionJobLauncherCommandLineRunnerTest.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/runner/EvaAccessionJobLauncherCommandLineRunnerTest.java index f07516469..660cef9cb 100644 --- a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/runner/EvaAccessionJobLauncherCommandLineRunnerTest.java +++ b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/runner/EvaAccessionJobLauncherCommandLineRunnerTest.java @@ -16,13 +16,13 @@ package uk.ac.ebi.eva.accession.pipeline.runner; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Ignore; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.batch.core.JobInstance; import org.springframework.batch.core.StepExecution; import org.springframework.batch.core.explore.JobExplorer; @@ -36,18 +36,20 @@ import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import org.springframework.test.web.client.ExpectedCount; import org.springframework.test.web.client.MockRestServiceServer; import org.springframework.web.client.RestTemplate; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; import uk.ac.ebi.eva.accession.core.runner.CommandLineRunnerUtils; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.pipeline.parameters.InputParameters; +import uk.ac.ebi.eva.accession.pipeline.test.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.accession.pipeline.test.BatchTestConfiguration; +import uk.ac.ebi.eva.accession.pipeline.test.MongoTestConfiguration; import uk.ac.ebi.eva.commons.batch.io.VcfReader; import uk.ac.ebi.eva.metrics.count.CountServiceParameters; -import javax.sql.DataSource; import java.io.File; import java.net.URI; import java.nio.file.Files; @@ -58,7 +60,7 @@ import java.util.Arrays; import java.util.Collections; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.springframework.test.web.client.match.MockRestRequestMatchers.method; import static org.springframework.test.web.client.match.MockRestRequestMatchers.requestTo; import static org.springframework.test.web.client.response.MockRestResponseCreators.withStatus; @@ -72,10 +74,11 @@ import static uk.ac.ebi.eva.accession.pipeline.runner.RunnerUtil.useTempVcfFile; import static uk.ac.ebi.eva.accession.pipeline.runner.RunnerUtil.writeToTempVCFFile; -@RunWith(SpringRunner.class) -@ContextConfiguration(classes = {BatchTestConfiguration.class}) +@ExtendWith(SpringExtension.class) +@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class, + BatchJobRepositoryTestConfiguration.class}) @TestPropertySource("classpath:accession-pipeline-test.properties") -public class EvaAccessionJobLauncherCommandLineRunnerTest { +public class EvaAccessionJobLauncherCommandLineRunnerTest extends MongoTestContainerHelper { @Autowired private InputParameters inputParameters; @@ -86,9 +89,6 @@ public class EvaAccessionJobLauncherCommandLineRunnerTest { @Autowired private JobExplorer jobExplorer; - @Autowired - private DataSource datasource; - @Autowired private EvaAccessionJobLauncherCommandLineRunner runner; @@ -123,18 +123,18 @@ public class EvaAccessionJobLauncherCommandLineRunnerTest { private boolean originalInputParametersCaptured = false; - @BeforeClass + @BeforeAll public static void initializeTempFile() throws Exception { tempVcfInputFileToTestFailingJobs = File.createTempFile("resumeFailingJob", ".vcf.gz"); tempVcfOutputDir = Files.createTempDirectory("contigs_variants_dir"); } - @AfterClass + @AfterAll public static void deleteTempFile() { tempVcfInputFileToTestFailingJobs.delete(); } - @Before + @BeforeEach public void setUp() throws Exception { if (!originalInputParametersCaptured) { originalVcfInputFilePath = inputParameters.getVcf(); @@ -143,8 +143,8 @@ public void setUp() throws Exception { writeToTempVCFFile(originalVcfContent, tempVcfInputFileToTestFailingJobs); originalInputParametersCaptured = true; } - jobRepositoryTestUtils = new JobRepositoryTestUtils(jobRepository, datasource); - runner.setJobNames(SUBSNP_ACCESSION_JOB); + jobRepositoryTestUtils = new JobRepositoryTestUtils(jobRepository); + runner.setJobName(SUBSNP_ACCESSION_JOB); deleteTemporaryContigAndVariantFiles(inputParameters, tempVcfOutputDir); useOriginalVcfFile(inputParameters, originalVcfInputFilePath, vcfReader); @@ -156,7 +156,7 @@ public void setUp() throws Exception { mongoTemplate.dropCollection(SubmittedVariantEntity.class); } - @After + @AfterEach public void tearDown() { jobRepositoryTestUtils.removeJobExecutions(); inputParameters.setForceRestart(false); @@ -171,7 +171,7 @@ public void runJobWithNoErrors() throws Exception { @Test public void runJobWithNoName() throws Exception { - runner.setJobNames(null); + runner.setJobName(null); runner.run(); assertEquals(EvaAccessionJobLauncherCommandLineRunner.EXIT_WITH_ERRORS, runner.getExitCode()); @@ -179,7 +179,7 @@ public void runJobWithNoName() throws Exception { @Test @DirtiesContext - @Ignore + @Disabled public void restartCompletedJobThatIsAlreadyInTheRepository() throws Exception { runner.run(); assertEquals(EvaAccessionJobLauncherCommandLineRunner.EXIT_WITHOUT_ERRORS, runner.getExitCode()); @@ -218,7 +218,7 @@ public void forceRestartButNoJobInTheRepository() throws Exception { @Test @DirtiesContext - @Ignore + @Disabled public void resumeFailingJobFromCorrectChunk() throws Exception { // Jobs A, B, C are run chronological order; A and C have SAME parameters; // A is the job that is run after VCF fault injection (as part of the runTestWithFaultInjection method), diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/runner/JobFailureBlocksReleasedTest.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/runner/JobFailureBlocksReleasedTest.java index d327f8077..cbbb5eebc 100644 --- a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/runner/JobFailureBlocksReleasedTest.java +++ b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/runner/JobFailureBlocksReleasedTest.java @@ -16,29 +16,32 @@ * limitations under the License. */ -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.ampt2d.commons.accession.persistence.jpa.monotonic.repositories.ContiguousIdBlockRepository; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.pipeline.parameters.InputParameters; +import uk.ac.ebi.eva.accession.pipeline.test.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.accession.pipeline.test.BatchTestConfiguration; +import uk.ac.ebi.eva.accession.pipeline.test.MongoTestConfiguration; import uk.ac.ebi.eva.commons.batch.io.VcfReader; import java.io.File; import java.nio.file.Files; import java.nio.file.Path; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.SUBSNP_ACCESSION_JOB; import static uk.ac.ebi.eva.accession.pipeline.runner.RunnerUtil.deleteTemporaryContigAndVariantFiles; import static uk.ac.ebi.eva.accession.pipeline.runner.RunnerUtil.getOriginalVcfContent; @@ -47,10 +50,11 @@ import static uk.ac.ebi.eva.accession.pipeline.runner.RunnerUtil.useTempVcfFile; import static uk.ac.ebi.eva.accession.pipeline.runner.RunnerUtil.writeToTempVCFFile; -@RunWith(SpringRunner.class) -@ContextConfiguration(classes = {BatchTestConfiguration.class}) +@ExtendWith(SpringExtension.class) +@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class, + BatchJobRepositoryTestConfiguration.class}) @TestPropertySource("classpath:accession-pipeline-test.properties") -public class JobFailureBlocksReleasedTest { +public class JobFailureBlocksReleasedTest extends MongoTestContainerHelper { @Autowired private InputParameters inputParameters; @@ -74,24 +78,24 @@ public class JobFailureBlocksReleasedTest { @Autowired private ContiguousIdBlockRepository blockRepository; - @BeforeClass + @BeforeAll public static void initializeTempFile() throws Exception { tempVcfInputFileToTestFailingJobs = File.createTempFile("resumeFailingJob", ".vcf.gz"); tempVcfOutputDir = Files.createTempDirectory("contigs_variants_dir"); } - @AfterClass + @AfterAll public static void deleteTempFile() { tempVcfInputFileToTestFailingJobs.delete(); } - @Before + @BeforeEach public void setUp() throws Exception { originalVcfInputFilePath = inputParameters.getVcf(); originalVcfContent = getOriginalVcfContent(originalVcfInputFilePath); writeToTempVCFFile(originalVcfContent, tempVcfInputFileToTestFailingJobs); - runner.setJobNames(SUBSNP_ACCESSION_JOB); + runner.setJobName(SUBSNP_ACCESSION_JOB); deleteTemporaryContigAndVariantFiles(inputParameters, tempVcfOutputDir); useOriginalVcfFile(inputParameters, originalVcfInputFilePath, vcfReader); diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/runner/RestartFailedJobTest.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/runner/RestartFailedJobTest.java index df57e760e..ea0cd06df 100644 --- a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/runner/RestartFailedJobTest.java +++ b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/runner/RestartFailedJobTest.java @@ -16,12 +16,12 @@ package uk.ac.ebi.eva.accession.pipeline.runner; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.batch.core.JobInstance; import org.springframework.batch.core.StepExecution; import org.springframework.batch.core.explore.JobExplorer; @@ -36,26 +36,28 @@ import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import org.springframework.test.web.client.ExpectedCount; import org.springframework.test.web.client.MockRestServiceServer; import org.springframework.web.client.RestTemplate; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; import uk.ac.ebi.eva.accession.core.runner.CommandLineRunnerUtils; import uk.ac.ebi.eva.accession.core.service.nonhuman.SubmittedVariantAccessioningService; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.pipeline.parameters.InputParameters; +import uk.ac.ebi.eva.accession.pipeline.test.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.accession.pipeline.test.BatchTestConfiguration; +import uk.ac.ebi.eva.accession.pipeline.test.MongoTestConfiguration; import uk.ac.ebi.eva.commons.batch.io.VcfReader; import uk.ac.ebi.eva.metrics.count.CountServiceParameters; -import javax.sql.DataSource; import java.io.File; import java.net.URI; import java.nio.file.Files; import java.nio.file.Path; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.mockito.Mockito.doNothing; import static org.springframework.test.web.client.match.MockRestRequestMatchers.method; import static org.springframework.test.web.client.match.MockRestRequestMatchers.requestTo; @@ -70,10 +72,11 @@ import static uk.ac.ebi.eva.accession.pipeline.runner.RunnerUtil.useTempVcfFile; import static uk.ac.ebi.eva.accession.pipeline.runner.RunnerUtil.writeToTempVCFFile; -@RunWith(SpringRunner.class) -@ContextConfiguration(classes = {BatchTestConfiguration.class}) +@ExtendWith(SpringExtension.class) +@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class, + BatchJobRepositoryTestConfiguration.class}) @TestPropertySource("classpath:accession-pipeline-test.properties") -public class RestartFailedJobTest { +public class RestartFailedJobTest extends MongoTestContainerHelper { @Autowired private InputParameters inputParameters; @@ -84,9 +87,6 @@ public class RestartFailedJobTest { @Autowired private JobExplorer jobExplorer; - @Autowired - private DataSource datasource; - @Autowired private EvaAccessionJobLauncherCommandLineRunner runner; @@ -115,8 +115,6 @@ public class RestartFailedJobTest { private static String originalVcfInputFilePath; - private static String originalVcfOutputFilePath; - private static String originalVcfContent; private boolean originalInputParametersCaptured = false; @@ -124,28 +122,27 @@ public class RestartFailedJobTest { @SpyBean private SubmittedVariantAccessioningService accessioningServiceSpy; - @BeforeClass + @BeforeAll public static void initializeTempFile() throws Exception { tempVcfInputFileToTestFailingJobs = File.createTempFile("resumeFailingJob", ".vcf.gz"); tempVcfOutputDir = Files.createTempDirectory("contigs_variants_dir"); } - @AfterClass + @AfterAll public static void deleteTempFile() { tempVcfInputFileToTestFailingJobs.delete(); } - @Before + @BeforeEach public void setUp() throws Exception { if (!originalInputParametersCaptured) { originalVcfInputFilePath = inputParameters.getVcf(); - originalVcfOutputFilePath = inputParameters.getOutputVcf(); originalVcfContent = getOriginalVcfContent(originalVcfInputFilePath); writeToTempVCFFile(originalVcfContent, tempVcfInputFileToTestFailingJobs); originalInputParametersCaptured = true; } - jobRepositoryTestUtils = new JobRepositoryTestUtils(jobRepository, datasource); - runner.setJobNames(SUBSNP_ACCESSION_JOB); + jobRepositoryTestUtils = new JobRepositoryTestUtils(jobRepository); + runner.setJobName(SUBSNP_ACCESSION_JOB); deleteTemporaryContigAndVariantFiles(inputParameters, tempVcfOutputDir); useOriginalVcfFile(inputParameters, originalVcfInputFilePath, vcfReader); @@ -160,7 +157,7 @@ public void setUp() throws Exception { } - @After + @AfterEach public void tearDown() { jobRepositoryTestUtils.removeJobExecutions(); inputParameters.setForceRestart(false); diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/runner/SSAccessionRecoveryTest.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/runner/SSAccessionRecoveryTest.java index 16dbca827..dacbf692c 100644 --- a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/runner/SSAccessionRecoveryTest.java +++ b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/runner/SSAccessionRecoveryTest.java @@ -16,33 +16,37 @@ * limitations under the License. */ -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.batch.test.context.SpringBatchTest; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.ampt2d.commons.accession.persistence.jpa.monotonic.entities.ContiguousIdBlock; import uk.ac.ebi.ampt2d.commons.accession.persistence.jpa.monotonic.repositories.ContiguousIdBlockRepository; import uk.ac.ebi.eva.accession.core.model.SubmittedVariant; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; import uk.ac.ebi.eva.accession.core.repository.nonhuman.eva.SubmittedVariantAccessioningRepository; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; +import uk.ac.ebi.eva.accession.pipeline.test.BatchJobRepositoryTestConfiguration; +import uk.ac.ebi.eva.accession.pipeline.test.MongoTestConfiguration; import uk.ac.ebi.eva.accession.pipeline.test.SSAccessionRecoveryTestConfiguration; import java.util.ArrayList; import java.util.List; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.SS_ACCESSION_RECOVERY_JOB; -@RunWith(SpringRunner.class) -@ContextConfiguration(classes = {SSAccessionRecoveryTestConfiguration.class}) +@ExtendWith(SpringExtension.class) +@ContextConfiguration(classes = {SSAccessionRecoveryTestConfiguration.class, MongoTestConfiguration.class, + BatchJobRepositoryTestConfiguration.class}) @TestPropertySource("classpath:ss-accession-recovery.properties") @SpringBatchTest -public class SSAccessionRecoveryTest { +public class SSAccessionRecoveryTest extends MongoTestContainerHelper { @Autowired private EvaAccessionJobLauncherCommandLineRunner runner; @@ -59,7 +63,7 @@ public void testContiguousBlocksForCategorySSAreRecovered() throws Exception { verifyInitialDBState(); // recovery cut off time is -14 days (provided in ss-accession-recovery.properties) - runner.setJobNames(SS_ACCESSION_RECOVERY_JOB); + runner.setJobName(SS_ACCESSION_RECOVERY_JOB); runner.run(); assertEquals(EvaAccessionJobLauncherCommandLineRunner.EXIT_WITHOUT_ERRORS, runner.getExitCode()); diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/test/BatchJobRepositoryTestConfiguration.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/test/BatchJobRepositoryTestConfiguration.java new file mode 100644 index 000000000..776d64a87 --- /dev/null +++ b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/test/BatchJobRepositoryTestConfiguration.java @@ -0,0 +1,22 @@ +package uk.ac.ebi.eva.accession.pipeline.test; + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Primary; +import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseBuilder; +import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseType; + +import javax.sql.DataSource; + +@Configuration +public class BatchJobRepositoryTestConfiguration { + + @Bean + @Primary + public DataSource dataSource() { + return new EmbeddedDatabaseBuilder() + .setType(EmbeddedDatabaseType.HSQL) + .addScript("org/springframework/batch/core/schema-hsqldb.sql") + .build(); + } +} \ No newline at end of file diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/test/BatchTestConfiguration.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/test/BatchTestConfiguration.java index 132d38828..38a6459e5 100644 --- a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/test/BatchTestConfiguration.java +++ b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/test/BatchTestConfiguration.java @@ -17,15 +17,16 @@ package uk.ac.ebi.eva.accession.pipeline.test; import org.springframework.batch.core.Job; +import org.springframework.batch.core.launch.JobLauncher; +import org.springframework.batch.core.repository.JobRepository; import org.springframework.batch.test.JobLauncherTestUtils; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Import; import uk.ac.ebi.eva.accession.pipeline.configuration.batch.io.AccessionWriterConfiguration; -import uk.ac.ebi.eva.accession.pipeline.configuration.batch.io.SSAccFileReaderConfiguration; import uk.ac.ebi.eva.accession.pipeline.configuration.batch.io.DuplicateSSAccQCWriterConfiguration; +import uk.ac.ebi.eva.accession.pipeline.configuration.batch.io.SSAccFileReaderConfiguration; import uk.ac.ebi.eva.accession.pipeline.configuration.batch.io.VcfReaderConfiguration; import uk.ac.ebi.eva.accession.pipeline.configuration.batch.jobs.DuplicateSSAccQCJobConfiguration; import uk.ac.ebi.eva.accession.pipeline.configuration.batch.jobs.QCSubsnpAccessionsJobConfiguration; @@ -63,36 +64,33 @@ public class BatchTestConfiguration { public static final String JOB_LAUNCHER_DUPLICATE_SS_ACC_QC_JOB = "JOB_LAUNCHER_DUPLICATE_SS_ACC_QC_JOB"; @Bean(JOB_LAUNCHER_SUBSNP_ACCESSION_JOB) - public JobLauncherTestUtils jobLauncherTestUtilsCreate() { - return new JobLauncherTestUtils() { - @Override - @Autowired - public void setJob(@Qualifier(SUBSNP_ACCESSION_JOB) Job job) { - super.setJob(job); - } - }; + public JobLauncherTestUtils jobLauncherTestUtilsCreate(JobLauncher jobLauncher, JobRepository jobRepository, + @Qualifier(SUBSNP_ACCESSION_JOB) Job job) { + JobLauncherTestUtils utils = new JobLauncherTestUtils(); + utils.setJobLauncher(jobLauncher); + utils.setJobRepository(jobRepository); + utils.setJob(job); + return utils; } @Bean(JOB_LAUNCHER_QC_SUBSNP_ACCESSION_JOB) - public JobLauncherTestUtils jobLauncherTestUtilsQC() { - return new JobLauncherTestUtils() { - @Override - @Autowired - public void setJob(@Qualifier(QC_SUBSNP_ACCESSION_JOB) Job job) { - super.setJob(job); - } - }; + public JobLauncherTestUtils jobLauncherTestUtilsQC(JobLauncher jobLauncher, JobRepository jobRepository, + @Qualifier(QC_SUBSNP_ACCESSION_JOB) Job job) { + JobLauncherTestUtils utils = new JobLauncherTestUtils(); + utils.setJobLauncher(jobLauncher); + utils.setJobRepository(jobRepository); + utils.setJob(job); + return utils; } @Bean(JOB_LAUNCHER_DUPLICATE_SS_ACC_QC_JOB) - public JobLauncherTestUtils jobLauncherTestUtilsDuplicateSSAccQC() { - return new JobLauncherTestUtils() { - @Override - @Autowired - public void setJob(@Qualifier(DUPLICATE_SS_ACC_QC_JOB) Job job) { - super.setJob(job); - } - }; + public JobLauncherTestUtils jobLauncherTestUtilsDuplicateSSAccQC(JobLauncher jobLauncher, JobRepository jobRepository, + @Qualifier(DUPLICATE_SS_ACC_QC_JOB) Job job) { + JobLauncherTestUtils utils = new JobLauncherTestUtils(); + utils.setJobLauncher(jobLauncher); + utils.setJobRepository(jobRepository); + utils.setJob(job); + return utils; } @Bean diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/test/FixSpringMongoDbRule.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/test/FixSpringMongoDbRule.java deleted file mode 100644 index 0ff4b7dca..000000000 --- a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/test/FixSpringMongoDbRule.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * - * Copyright 2019 EMBL - European Bioinformatics Institute - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -package uk.ac.ebi.eva.accession.pipeline.test; - -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfiguration; -import com.lordofthejars.nosqlunit.mongodb.SpringMongoDbRule; - -/** - * Temporary fix until nosql unit rc-6 or final is released - */ -public class FixSpringMongoDbRule extends SpringMongoDbRule { - - public FixSpringMongoDbRule(MongoDbConfiguration mongoDbConfiguration) { - super(mongoDbConfiguration); - } - - public FixSpringMongoDbRule(MongoDbConfiguration mongoDbConfiguration, Object object) { - super(mongoDbConfiguration, object); - } - - @Override - public void close() { - // DO NOT CLOSE the connection (Spring will do it when destroying the context) - } - -} diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/test/MongoTestConfiguration.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/test/MongoTestConfiguration.java new file mode 100644 index 000000000..2a261f1b6 --- /dev/null +++ b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/test/MongoTestConfiguration.java @@ -0,0 +1,102 @@ +/* + * Copyright 2021 EMBL - European Bioinformatics Institute + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package uk.ac.ebi.eva.accession.pipeline.test; + +import com.mongodb.ReadPreference; +import com.mongodb.WriteConcern; +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoClients; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.domain.EntityScan; +import org.springframework.boot.test.autoconfigure.data.mongo.AutoConfigureDataMongo; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Primary; +import org.springframework.data.mongodb.MongoDatabaseFactory; +import org.springframework.data.mongodb.config.EnableMongoAuditing; +import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.data.mongodb.core.SimpleMongoClientDatabaseFactory; +import org.springframework.data.mongodb.core.WriteResultChecking; +import org.springframework.data.mongodb.core.convert.DefaultDbRefResolver; +import org.springframework.data.mongodb.core.convert.DefaultMongoTypeMapper; +import org.springframework.data.mongodb.core.convert.MappingMongoConverter; +import org.springframework.data.mongodb.core.convert.MongoCustomConversions; +import org.springframework.data.mongodb.core.mapping.MongoMappingContext; +import org.springframework.data.mongodb.repository.config.EnableMongoRepositories; + +import java.util.Collections; + +@Configuration +@EntityScan(basePackages = {"uk.ac.ebi.eva.accession.core.persistence"}) +@EnableMongoRepositories(basePackages = { + "uk.ac.ebi.eva.accession.core.persistence", + "uk.ac.ebi.ampt2d.commons.accession.persistence.mongodb.repository"}) +@EnableMongoAuditing +@AutoConfigureDataMongo +public class MongoTestConfiguration { + + @Value("${spring.data.mongodb.uri}") + private String mongoUri; + + @Value("${spring.data.mongodb.database}") + private String database; + + @Value("${mongodb.read-preference:primary}") + private String readPreference; + + @Bean + @Primary + public MongoClient mongoClient() { + return MongoClients.create(mongoUri); + } + + @Bean("primaryFactory") + @Primary + public MongoDatabaseFactory mongoDbFactory() { + return new SimpleMongoClientDatabaseFactory(mongoClient(), database); + } + + @Bean + @Primary + public MongoCustomConversions mongoCustomConversions() { + return new MongoCustomConversions(Collections.emptyList()); + } + + @Bean + @Primary + public MongoMappingContext mongoMappingContext(MongoCustomConversions conversions) { + MongoMappingContext context = new MongoMappingContext(); + context.setSimpleTypeHolder(conversions.getSimpleTypeHolder()); + context.setAutoIndexCreation(false); + return context; + } + + @Bean + @Primary + public MongoTemplate mongoTemplate(MongoMappingContext mongoMappingContext, + MongoCustomConversions conversions) { + MappingMongoConverter converter = new MappingMongoConverter( + new DefaultDbRefResolver(mongoDbFactory()), mongoMappingContext); + converter.setCustomConversions(conversions); + converter.setTypeMapper(new DefaultMongoTypeMapper(null)); + converter.afterPropertiesSet(); + MongoTemplate mongoTemplate = new MongoTemplate(mongoDbFactory(), converter); + mongoTemplate.setReadPreference(ReadPreference.valueOf(readPreference)); + mongoTemplate.setWriteConcern(WriteConcern.MAJORITY); + mongoTemplate.setWriteResultChecking(WriteResultChecking.EXCEPTION); + return mongoTemplate; + } +} diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/test/SSAccessionRecoveryTestConfiguration.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/test/SSAccessionRecoveryTestConfiguration.java index 4e9d3a2b5..4556a4439 100644 --- a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/test/SSAccessionRecoveryTestConfiguration.java +++ b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/test/SSAccessionRecoveryTestConfiguration.java @@ -16,7 +16,8 @@ package uk.ac.ebi.eva.accession.pipeline.test; -import org.springframework.batch.core.configuration.annotation.BatchConfigurer; +import org.springframework.batch.core.launch.JobLauncher; +import org.springframework.batch.core.repository.JobRepository; import org.springframework.batch.test.JobLauncherTestUtils; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.context.annotation.Bean; @@ -28,12 +29,8 @@ import uk.ac.ebi.eva.accession.pipeline.configuration.batch.recovery.SSAccessionRecoveryServiceConfiguration; import uk.ac.ebi.eva.accession.pipeline.configuration.batch.steps.SSAccessionRecoveryStepConfiguration; import uk.ac.ebi.eva.accession.pipeline.runner.EvaAccessionJobLauncherCommandLineRunner; -import uk.ac.ebi.eva.commons.batch.configuration.SpringBoot1CompatibilityConfiguration; import uk.ac.ebi.eva.commons.batch.job.JobExecutionApplicationListener; -import javax.persistence.EntityManagerFactory; -import javax.sql.DataSource; - @EnableAutoConfiguration @Import({SSAccessionRecoveryJobConfiguration.class, SSAccessionRecoveryStepConfiguration.class, @@ -43,17 +40,10 @@ EvaAccessionJobLauncherCommandLineRunner.class}) public class SSAccessionRecoveryTestConfiguration { @Bean - public BatchConfigurer configurer(DataSource dataSource, EntityManagerFactory entityManagerFactory) - throws Exception { - return SpringBoot1CompatibilityConfiguration.getSpringBoot1CompatibleBatchConfigurer(dataSource, - entityManagerFactory); - } - - @Bean - public JobLauncherTestUtils jobLauncherTestUtils(BatchConfigurer configurer) throws Exception { + public JobLauncherTestUtils jobLauncherTestUtils(JobLauncher jobLauncher, JobRepository jobRepository) { JobLauncherTestUtils jobLauncherTestUtils = new JobLauncherTestUtils(); - jobLauncherTestUtils.setJobLauncher(configurer.getJobLauncher()); - jobLauncherTestUtils.setJobRepository(configurer.getJobRepository()); + jobLauncherTestUtils.setJobLauncher(jobLauncher); + jobLauncherTestUtils.setJobRepository(jobRepository); return jobLauncherTestUtils; } diff --git a/eva-accession-pipeline/src/test/resources/properties/accession-pipeline-interval-test.properties b/eva-accession-pipeline/src/test/resources/properties/accession-pipeline-interval-test.properties index 2c6b400e8..b30fb3297 100644 --- a/eva-accession-pipeline/src/test/resources/properties/accession-pipeline-interval-test.properties +++ b/eva-accession-pipeline/src/test/resources/properties/accession-pipeline-interval-test.properties @@ -23,9 +23,6 @@ parameters.contigNaming=SEQUENCE_NAME spring.jpa.show-sql=true spring.data.mongodb.database=test-db -spring.data.mongodb.host=|eva.mongo.host.test| -spring.data.mongodb.password= -spring.data.mongodb.port=27017 mongodb.read-preference=primary # See https://github.com/spring-projects/spring-boot/wiki/Spring-Boot-2.1-Release-Notes#bean-overriding diff --git a/eva-accession-pipeline/src/test/resources/properties/accession-pipeline-recover-state-test.properties b/eva-accession-pipeline/src/test/resources/properties/accession-pipeline-recover-state-test.properties index 8062413e2..2edb00288 100644 --- a/eva-accession-pipeline/src/test/resources/properties/accession-pipeline-recover-state-test.properties +++ b/eva-accession-pipeline/src/test/resources/properties/accession-pipeline-recover-state-test.properties @@ -8,9 +8,10 @@ spring.datasource.driver-class-name=org.hsqldb.jdbcDriver spring.datasource.url=jdbc:hsqldb:mem:db;sql.syntax_pgs=true;DB_CLOSE_DELAY=-1 spring.datasource.username=SA spring.datasource.password= -spring.datasource.schema=test-data/contiguous_id_blocks_schema.sql -spring.datasource.data=test-data/contiguous_id_blocks_recover_state_data.sql -spring.jpa.hibernate.ddl-auto=update +spring.sql.init.schema-locations=classpath:test-data/contiguous_id_blocks_schema.sql +spring.sql.init.data-locations=classpath:test-data/contiguous_id_blocks_recover_state_data.sql +spring.sql.init.mode=always +spring.jpa.hibernate.ddl-auto=none parameters.assemblyAccession=assembly parameters.taxonomyAccession=1111 @@ -30,11 +31,7 @@ eva.count-stats.password=password spring.jpa.show-sql=true -spring.data.mongodb.uri=mongodb://|eva.mongo.host.test|:27017 spring.data.mongodb.database=test-db -spring.data.mongodb.host=|eva.mongo.host.test| -spring.data.mongodb.password= -spring.data.mongodb.port=27017 mongodb.read-preference=primary # See https://github.com/spring-projects/spring-boot/wiki/Spring-Boot-2.1-Release-Notes#bean-overriding diff --git a/eva-accession-pipeline/src/test/resources/properties/accession-pipeline-recover-test.properties b/eva-accession-pipeline/src/test/resources/properties/accession-pipeline-recover-test.properties index ca993b4bb..26d8215a4 100644 --- a/eva-accession-pipeline/src/test/resources/properties/accession-pipeline-recover-test.properties +++ b/eva-accession-pipeline/src/test/resources/properties/accession-pipeline-recover-test.properties @@ -12,9 +12,10 @@ spring.datasource.driver-class-name=org.hsqldb.jdbcDriver spring.datasource.url=jdbc:hsqldb:mem:db;sql.syntax_pgs=true;DB_CLOSE_DELAY=-1 spring.datasource.username=SA spring.datasource.password= -spring.datasource.schema=test-data/contiguous_id_blocks_schema.sql -spring.datasource.data=test-data/contiguous_id_blocks_data.sql -spring.jpa.hibernate.ddl-auto=update +spring.sql.init.schema-locations=classpath:test-data/contiguous_id_blocks_schema.sql +spring.sql.init.data-locations=classpath:test-data/contiguous_id_blocks_data.sql +spring.sql.init.mode=always +spring.jpa.hibernate.ddl-auto=none parameters.assemblyAccession=assembly parameters.taxonomyAccession=1111 @@ -30,11 +31,7 @@ parameters.contigNaming=SEQUENCE_NAME spring.jpa.show-sql=true -spring.data.mongodb.uri=mongodb://|eva.mongo.host.test|:27017 spring.data.mongodb.database=test-db -spring.data.mongodb.host=|eva.mongo.host.test| -spring.data.mongodb.password= -spring.data.mongodb.port=27017 mongodb.read-preference=primary # See https://github.com/spring-projects/spring-boot/wiki/Spring-Boot-2.1-Release-Notes#bean-overriding diff --git a/eva-accession-pipeline/src/test/resources/properties/accession-pipeline-test.properties b/eva-accession-pipeline/src/test/resources/properties/accession-pipeline-test.properties index 7ad05b90b..5a20cf25b 100644 --- a/eva-accession-pipeline/src/test/resources/properties/accession-pipeline-test.properties +++ b/eva-accession-pipeline/src/test/resources/properties/accession-pipeline-test.properties @@ -15,6 +15,7 @@ parameters.projectAccession=project parameters.chunkSize=5 parameters.vcf=src/test/resources/input-files/vcf/small_genotyped.vcf.gz parameters.vcfAggregation=NONE +parameters.aggregatedMappingFile=src/test/resources/output_files/aggregated_mapping_file.txt parameters.fasta=src/test/resources/input-files/fasta/Homo_sapiens.GRCh37.75.chr20.head_1200.fa parameters.outputVcf=/tmp/accession-output.vcf @@ -24,9 +25,6 @@ parameters.contigNaming=SEQUENCE_NAME spring.jpa.show-sql=true spring.data.mongodb.database=test-db -spring.data.mongodb.host=|eva.mongo.host.test| -spring.data.mongodb.password= -spring.data.mongodb.port=27017 mongodb.read-preference=primary # See https://github.com/spring-projects/spring-boot/wiki/Spring-Boot-2.1-Release-Notes#bean-overriding diff --git a/eva-accession-pipeline/src/test/resources/properties/duplicate-ss-acc-qc-test.properties b/eva-accession-pipeline/src/test/resources/properties/duplicate-ss-acc-qc-test.properties index 3a37481af..183cdbf90 100644 --- a/eva-accession-pipeline/src/test/resources/properties/duplicate-ss-acc-qc-test.properties +++ b/eva-accession-pipeline/src/test/resources/properties/duplicate-ss-acc-qc-test.properties @@ -25,9 +25,6 @@ parameters.contigNaming=SEQUENCE_NAME spring.jpa.show-sql=true spring.data.mongodb.database=test-db -spring.data.mongodb.host=|eva.mongo.host.test| -spring.data.mongodb.password= -spring.data.mongodb.port=27017 mongodb.read-preference=primary # See https://github.com/spring-projects/spring-boot/wiki/Spring-Boot-2.1-Release-Notes#bean-overriding diff --git a/eva-accession-pipeline/src/test/resources/properties/qc-accession-pipeline-test.properties b/eva-accession-pipeline/src/test/resources/properties/qc-accession-pipeline-test.properties index da55ada11..28c3df942 100644 --- a/eva-accession-pipeline/src/test/resources/properties/qc-accession-pipeline-test.properties +++ b/eva-accession-pipeline/src/test/resources/properties/qc-accession-pipeline-test.properties @@ -1,5 +1,9 @@ accessioning.submitted.categoryId=test-pipeline-ss +eva.count-stats.url=http://localhost:8080 +eva.count-stats.username=username +eva.count-stats.password=password + accessioning.monotonic.test-pipeline-ss.blockSize=100000 accessioning.monotonic.test-pipeline-ss.blockStartValue=5000000000 accessioning.monotonic.test-pipeline-ss.nextBlockInterval=1000000000 @@ -19,9 +23,6 @@ parameters.contigNaming=SEQUENCE_NAME spring.jpa.show-sql=true spring.data.mongodb.database=test-db -spring.data.mongodb.host=|eva.mongo.host.test| -spring.data.mongodb.password= -spring.data.mongodb.port=27017 mongodb.read-preference=primary # See https://github.com/spring-projects/spring-boot/wiki/Spring-Boot-2.1-Release-Notes#bean-overriding diff --git a/eva-accession-pipeline/src/test/resources/properties/ss-accession-recovery.properties b/eva-accession-pipeline/src/test/resources/properties/ss-accession-recovery.properties index a16c85174..343cf5fde 100644 --- a/eva-accession-pipeline/src/test/resources/properties/ss-accession-recovery.properties +++ b/eva-accession-pipeline/src/test/resources/properties/ss-accession-recovery.properties @@ -4,9 +4,10 @@ spring.datasource.driver-class-name=org.hsqldb.jdbcDriver spring.datasource.url=jdbc:hsqldb:mem:db;sql.syntax_pgs=true;DB_CLOSE_DELAY=-1 spring.datasource.username=SA spring.datasource.password= -spring.datasource.schema=test-data/contiguous_id_blocks_schema.sql -spring.datasource.data=test-data/ss_accession_recovery_test_data.sql -spring.jpa.hibernate.ddl-auto=update +spring.sql.init.schema-locations=classpath:test-data/contiguous_id_blocks_schema.sql +spring.sql.init.data-locations=classpath:test-data/ss_accession_recovery_test_data.sql +spring.sql.init.mode=always +spring.jpa.hibernate.ddl-auto=none accessioning.submitted.categoryId=test-pipeline-ss @@ -24,6 +25,7 @@ parameters.projectAccession=project parameters.chunkSize=5 parameters.vcf=src/test/resources/input-files/vcf/small_genotyped.vcf.gz parameters.vcfAggregation=NONE +parameters.aggregatedMappingFile=src/test/resources/output_files/aggregated_mapping_file.txt parameters.fasta=src/test/resources/input-files/fasta/Homo_sapiens.GRCh37.75.chr20.head_1200.fa parameters.outputVcf=/tmp/accession-output.vcf @@ -33,9 +35,6 @@ parameters.contigNaming=SEQUENCE_NAME spring.jpa.show-sql=true spring.data.mongodb.database=test-db -spring.data.mongodb.host=|eva.mongo.host.test| -spring.data.mongodb.password= -spring.data.mongodb.port=27017 mongodb.read-preference=primary # See https://github.com/spring-projects/spring-boot/wiki/Spring-Boot-2.1-Release-Notes#bean-overriding diff --git a/eva-accession-release/pom.xml b/eva-accession-release/pom.xml index be9db836f..dfc1ce525 100644 --- a/eva-accession-release/pom.xml +++ b/eva-accession-release/pom.xml @@ -2,12 +2,13 @@ + 4.0.0 + eva-accession uk.ac.ebi.eva - 0.6.58-SNAPSHOT + ${revision} - 4.0.0 eva-accession-release jar @@ -25,7 +26,7 @@ uk.ac.ebi.eva eva-accession-core test-jar - ${project.version} + ${revision} test @@ -42,14 +43,29 @@ xstream + + org.glassfish.jaxb + jaxb-runtime + + org.springframework.batch spring-batch-test test - com.lordofthejars - nosqlunit-mongodb + org.springframework.boot + spring-boot-testcontainers + test + + + org.testcontainers + mongodb + test + + + org.testcontainers + junit-jupiter test @@ -89,7 +105,4 @@ - - localhost - diff --git a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/assembly/AssemblyNameRetriever.java b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/assembly/AssemblyNameRetriever.java index e5082d300..cbcb55138 100644 --- a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/assembly/AssemblyNameRetriever.java +++ b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/assembly/AssemblyNameRetriever.java @@ -15,11 +15,12 @@ */ package uk.ac.ebi.eva.accession.release.assembly; -import javax.xml.bind.JAXBContext; -import javax.xml.bind.JAXBException; -import javax.xml.bind.Unmarshaller; -import javax.xml.bind.annotation.XmlElement; -import javax.xml.bind.annotation.XmlRootElement; +import jakarta.xml.bind.JAXBContext; +import jakarta.xml.bind.JAXBException; +import jakarta.xml.bind.Unmarshaller; +import jakarta.xml.bind.annotation.XmlElement; +import jakarta.xml.bind.annotation.XmlRootElement; + import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; @@ -35,7 +36,7 @@ /** * Use a curated list and the ENA webservices to return the assembly name associated with an assembly accession. - * + *

* The ENA webservices for URLs like https://www.ebi.ac.uk/ena/browser/api/xml/GCA_000001405.28 * can return an xml that starts like this: * 

@@ -47,7 +48,7 @@
  * ...
  * }
  *
- * + *

* The schema for these XMLs (and the definition of the NAME element) is here: * https://github.com/enasequence/schema/blob/master/src/main/resources/uk/ac/ebi/ena/sra/schema/ENA.assembly.xsd#L44 */ @@ -96,7 +97,7 @@ private Optional fetchAssemblyName(String assemblyAccession) { String errorMessage = new BufferedReader(new InputStreamReader(httpConnection.getErrorStream())) .lines().collect(Collectors.joining("\n")); throw new RuntimeException("Unexpected response (HTTP code " + responseCode + "). Message: " - + errorMessage); + + errorMessage); } } catch (IOException | JAXBException e) { throw new RuntimeException(e); diff --git a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/VariantMongoAggregationReader.java b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/VariantMongoAggregationReader.java index d1f739cb3..a6b1c28a5 100644 --- a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/VariantMongoAggregationReader.java +++ b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/VariantMongoAggregationReader.java @@ -16,8 +16,8 @@ package uk.ac.ebi.eva.accession.release.batch.io; -import com.mongodb.MongoClient; import com.mongodb.client.AggregateIterable; +import com.mongodb.client.MongoClient; import com.mongodb.client.MongoCollection; import com.mongodb.client.MongoCursor; import com.mongodb.client.MongoDatabase; @@ -131,9 +131,8 @@ protected void aggregate(String collectionName) { logger.info("issuing aggregation on collection {}", collectionName); MongoCollection collection = db.getCollection(collectionName); AggregateIterable clusteredVariants = collection.aggregate(buildAggregation()) - .allowDiskUse(true) - .useCursor(true) - .batchSize(chunkSize); + .allowDiskUse(true) + .batchSize(chunkSize); cursor = clusteredVariants.iterator(); } @@ -151,8 +150,8 @@ protected VariantSourceEntry buildVariantSourceEntry(String study, String sequen boolean assemblyMatch, boolean evidence, boolean remappedRS, Long mergedInto) { VariantSourceEntry variantSourceEntry = buildVariantSourceEntry(study, sequenceOntology, validated, - submittedVariantValidated, allelesMatch, - assemblyMatch, evidence, remappedRS); + submittedVariantValidated, allelesMatch, + assemblyMatch, evidence, remappedRS); if (Objects.nonNull(mergedInto)) { variantSourceEntry.addAttribute(MERGED_INTO_KEY, buildId(mergedInto)); } @@ -223,15 +222,15 @@ protected boolean isSameLocation(String contig, long start, String submittedVari * The start is considered to be the same when: * - start in clustered and submitted variant match * - start in clustered and submitted variant have a difference of 1 - * + *

* The start position can be different in ambiguous INDELS because the renormalization is only applied to * submitted variants. In those cases the start in the clustered and submitted variants will not exactly match but * the difference should be 1 - * + *

* Example: * RS (assembly: GCA_000309985.1, accession: 268233057, chromosome: CM001642.1, start: 7356605, type: INS) * SS (assembly: GCA_000309985.1, accession: 490570267, chromosome: CM001642.1, start: 7356604, reference: , - * alternate: AGAGCTATGATCTTCGGAAGGAGAAGGAGAAGGAAAAGATTCATGACGTCCAC) + * alternate: AGAGCTATGATCTTCGGAAGGAGAAGGAGAAGGAAAAGATTCATGACGTCCAC) */ private boolean isSameStart(long clusteredVariantStart, long submittedVariantStart, String type) { return clusteredVariantStart == submittedVariantStart diff --git a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/active/AccessionedVariantMongoReader.java b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/active/AccessionedVariantMongoReader.java index 3151ac166..7dd435b48 100644 --- a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/active/AccessionedVariantMongoReader.java +++ b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/active/AccessionedVariantMongoReader.java @@ -16,7 +16,7 @@ package uk.ac.ebi.eva.accession.release.batch.io.active; -import com.mongodb.MongoClient; +import com.mongodb.client.MongoClient; import com.mongodb.client.model.Aggregates; import com.mongodb.client.model.Field; import com.mongodb.client.model.Filters; diff --git a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/active/VariantContextWriter.java b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/active/VariantContextWriter.java index 63116225b..13afd5f10 100644 --- a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/active/VariantContextWriter.java +++ b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/active/VariantContextWriter.java @@ -25,10 +25,10 @@ import htsjdk.variant.vcf.VCFInfoHeaderLine; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.batch.item.Chunk; import org.springframework.batch.item.ExecutionContext; import org.springframework.batch.item.ItemStreamException; import org.springframework.batch.item.ItemStreamWriter; - import uk.ac.ebi.eva.accession.release.assembly.AssemblyNameRetriever; import java.io.BufferedReader; @@ -37,7 +37,6 @@ import java.io.IOException; import java.nio.file.Path; import java.util.HashSet; -import java.util.List; import java.util.Optional; import java.util.Set; @@ -52,7 +51,7 @@ /** * Writes a VCF file for the release of RS IDs mapped against a reference assembly - * + *

* To include the contigs in the meta section it reads the file generated in the previous step * {@link ListContigsStepConfiguration} */ @@ -96,31 +95,31 @@ protected Set buildHeaderLines() { addContigs(metaData); metaData.add(new VCFHeaderLine("reference", getReferenceAssemblyLine())); metaData.add(new VCFInfoHeaderLine(VARIANT_CLASS_KEY, 1, VCFHeaderLineType.String, - "Variant class according to the Sequence Ontology")); + "Variant class according to the Sequence Ontology")); metaData.add(new VCFInfoHeaderLine(STUDY_ID_KEY, VCFHeaderLineCount.UNBOUNDED, VCFHeaderLineType.String, - "Identifiers of studies that report a variant")); + "Identifiers of studies that report a variant")); metaData.add(new VCFInfoHeaderLine(CLUSTERED_VARIANT_VALIDATED_KEY, 0, VCFHeaderLineType.Flag, - "RS validated flag, present when the RS was validated by any method " - + "as indicated by the dbSNP validation status")); + "RS validated flag, present when the RS was validated by any method " + + "as indicated by the dbSNP validation status")); metaData.add(new VCFInfoHeaderLine(SUBMITTED_VARIANT_VALIDATED_KEY, 1, VCFHeaderLineType.Integer, - "Number of submitted variants clustered in an RS that were validated by any" - + " method as indicated by the dbSNP validation status")); + "Number of submitted variants clustered in an RS that were validated by any" + + " method as indicated by the dbSNP validation status")); metaData.add(new VCFInfoHeaderLine(ALLELES_MATCH_KEY, 0, VCFHeaderLineType.Flag, - "Alleles mismatch flag, present when some of the submitted variants have " - + "inconsistent allele information. See https://github" - + ".com/EBIvariation/eva-accession/wiki/Import-accessions-from-dbSNP" - + "#alleles-match")); + "Alleles mismatch flag, present when some of the submitted variants have " + + "inconsistent allele information. See https://github" + + ".com/EBIvariation/eva-accession/wiki/Import-accessions-from-dbSNP" + + "#alleles-match")); metaData.add(new VCFInfoHeaderLine(ASSEMBLY_MATCH_KEY, 0, VCFHeaderLineType.Flag, - "Assembly mismatch flag, present when the reference allele doesn't match " - + "the reference sequence")); + "Assembly mismatch flag, present when the reference allele doesn't match " + + "the reference sequence")); metaData.add(new VCFInfoHeaderLine(SUPPORTED_BY_EVIDENCE_KEY, 0, VCFHeaderLineType.Flag, - "Lack of evidence flag, present if no submitted variant includes genotype " - + "or frequency information")); + "Lack of evidence flag, present if no submitted variant includes genotype " + + "or frequency information")); metaData.add(new VCFInfoHeaderLine(REMAPPED_KEY, 0, VCFHeaderLineType.Flag, - "Remapped flag, present if the clustered variant originate only " - + "from submitted variants remapped to this assembly")); + "Remapped flag, present if the clustered variant originate only " + + "from submitted variants remapped to this assembly")); return metaData; } @@ -152,7 +151,7 @@ private void addContigs(Set metaData) { } @Override - public void write(List variantContexts) throws Exception { + public void write(Chunk variantContexts) throws Exception { for (VariantContext variantContext : variantContexts) { writer.add(variantContext); } diff --git a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/contig/ContigMongoReader.java b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/contig/ContigMongoReader.java index 4c8487664..4cae6d04f 100644 --- a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/contig/ContigMongoReader.java +++ b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/contig/ContigMongoReader.java @@ -16,8 +16,8 @@ package uk.ac.ebi.eva.accession.release.batch.io.contig; -import com.mongodb.MongoClient; import com.mongodb.client.AggregateIterable; +import com.mongodb.client.MongoClient; import com.mongodb.client.MongoCollection; import com.mongodb.client.MongoCursor; import com.mongodb.client.MongoDatabase; @@ -34,14 +34,13 @@ import org.springframework.batch.item.ParseException; import org.springframework.batch.item.UnexpectedInputException; import uk.ac.ebi.ampt2d.commons.accession.core.models.EventType; - import uk.ac.ebi.eva.accession.release.collectionNames.CollectionNames; import java.util.Arrays; import java.util.List; -import static uk.ac.ebi.eva.accession.release.batch.io.multimap.MultimapVariantMongoReader.NON_SINGLE_LOCATION_MAPPING; import static uk.ac.ebi.eva.accession.release.batch.io.multimap.MultimapVariantMongoReader.MAPPING_WEIGHT_FIELD; +import static uk.ac.ebi.eva.accession.release.batch.io.multimap.MultimapVariantMongoReader.NON_SINGLE_LOCATION_MAPPING; public class ContigMongoReader implements ItemStreamReader { @@ -83,22 +82,22 @@ public static ContigMongoReader activeContigReader(String assemblyAccession, int MongoClient mongoClient, String database, CollectionNames names) { return new ContigMongoReader(assemblyAccession, mongoClient, database, - names.getSubmittedVariantEntity(), - buildAggregationForActiveContigs(assemblyAccession, taxonomyAccession)); + names.getSubmittedVariantEntity(), + buildAggregationForActiveContigs(assemblyAccession, taxonomyAccession)); } public static ContigMongoReader mergedContigReader(String assemblyAccession, MongoClient mongoClient, String database, CollectionNames names) { return new ContigMongoReader(assemblyAccession, mongoClient, database, - names.getClusteredVariantOperationEntity(), - buildAggregationForMergedContigs(assemblyAccession)); + names.getClusteredVariantOperationEntity(), + buildAggregationForMergedContigs(assemblyAccession)); } public static ContigMongoReader multimapContigReader(String assemblyAccession, MongoClient mongoClient, String database, CollectionNames names) { return new ContigMongoReader(assemblyAccession, mongoClient, database, - names.getClusteredVariantEntity(), - buildAggregationForMultimapContigs(assemblyAccession)); + names.getClusteredVariantEntity(), + buildAggregationForMultimapContigs(assemblyAccession)); } private ContigMongoReader(String assemblyAccession, MongoClient mongoClient, String database, String collection, @@ -122,20 +121,20 @@ private static List buildAggregationForActiveContigs(String assemblyAccess /** * Apparently, a $group aggregation stage yields a different result * if you do the $group on a nested field like "inactiveObjects.contig": - * + *

* { "_id" : [ "KB882311.1" ] } - * + *

* while if the $group is done on a simple toplevel field it returns - * + *

* { "_id" : "KB882311.1" } - * + *

* so, as a $group on an array is not what we want to do, we have to * $project the contig field into a toplevel field * before we can do the $group. */ private static List buildAggregationForMergedContigs(String assemblyAccession) { Bson match = Aggregates.match(Filters.and(Filters.eq(INACTIVE_REFERENCE_ASSEMBLY_FIELD, assemblyAccession), - Filters.eq(EVENT_TYPE_FIELD, EventType.MERGED.toString()))); + Filters.eq(EVENT_TYPE_FIELD, EventType.MERGED.toString()))); Bson extractContig = Aggregates.project(new Document(MONGO_ID_FIELD, INACTIVE_CONTIG_KEY)); @@ -151,7 +150,7 @@ private static List buildAggregationForMergedContigs(String assemblyAccess private static List buildAggregationForMultimapContigs(String assemblyAccession) { Bson match = Aggregates.match(Filters.and(Filters.eq(ACTIVE_REFERENCE_ASSEMBLY_FIELD, assemblyAccession), - Filters.gte(MAPPING_WEIGHT_FIELD, NON_SINGLE_LOCATION_MAPPING))); + Filters.gte(MAPPING_WEIGHT_FIELD, NON_SINGLE_LOCATION_MAPPING))); Bson uniqueContigs = Aggregates.group(ACTIVE_CONTIG_KEY); List aggregation = Arrays.asList(match, uniqueContigs); logger.info("Issuing aggregation: {}", aggregation); @@ -168,8 +167,7 @@ private void aggregate() { MongoDatabase db = mongoClient.getDatabase(database); MongoCollection mongoCollection = db.getCollection(collection); AggregateIterable clusteredVariants = mongoCollection.aggregate(aggregation) - .allowDiskUse(true) - .useCursor(true); + .allowDiskUse(true); cursor = clusteredVariants.iterator(); } diff --git a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/contig/ContigWriter.java b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/contig/ContigWriter.java index 853cee24d..a676030b8 100644 --- a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/contig/ContigWriter.java +++ b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/contig/ContigWriter.java @@ -15,10 +15,10 @@ */ package uk.ac.ebi.eva.accession.release.batch.io.contig; +import org.springframework.batch.item.Chunk; import org.springframework.batch.item.ExecutionContext; import org.springframework.batch.item.ItemStreamException; import org.springframework.batch.item.ItemStreamWriter; - import uk.ac.ebi.eva.accession.core.contig.ContigMapping; import uk.ac.ebi.eva.accession.core.contig.ContigNaming; import uk.ac.ebi.eva.accession.core.contig.ContigSynonyms; @@ -28,7 +28,6 @@ import java.io.IOException; import java.io.PrintWriter; import java.nio.file.Paths; -import java.util.List; /** * Writes the list of contigs to a flat file @@ -78,7 +77,7 @@ public void close() throws ItemStreamException { } @Override - public void write(List contigs) { + public void write(Chunk contigs) { for (String contig : contigs) { if (contig == null || contig.isEmpty()) { throw new IllegalArgumentException("The contig cannot be null or empty"); diff --git a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/deprecated/DeprecatedVariantAccessionWriter.java b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/deprecated/DeprecatedVariantAccessionWriter.java index 3e3362dba..2d4666268 100644 --- a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/deprecated/DeprecatedVariantAccessionWriter.java +++ b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/deprecated/DeprecatedVariantAccessionWriter.java @@ -15,10 +15,10 @@ */ package uk.ac.ebi.eva.accession.release.batch.io.deprecated; +import org.springframework.batch.item.Chunk; import org.springframework.batch.item.ExecutionContext; import org.springframework.batch.item.ItemStreamException; import org.springframework.batch.item.ItemStreamWriter; - import uk.ac.ebi.eva.commons.core.models.pipeline.Variant; import java.io.File; @@ -26,7 +26,6 @@ import java.io.IOException; import java.io.PrintWriter; import java.nio.file.Path; -import java.util.List; import java.util.Objects; /** @@ -66,8 +65,8 @@ public void close() throws ItemStreamException { } @Override - public void write(List variants) throws Exception { - for (Variant variant: variants) { + public void write(Chunk variants) throws Exception { + for (Variant variant : variants) { if (!Objects.isNull(variant)) { printWriter.println(variant.getMainId()); } diff --git a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/deprecated/DeprecatedVariantMongoReader.java b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/deprecated/DeprecatedVariantMongoReader.java index 041ce166f..eb9380a34 100644 --- a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/deprecated/DeprecatedVariantMongoReader.java +++ b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/deprecated/DeprecatedVariantMongoReader.java @@ -15,7 +15,7 @@ */ package uk.ac.ebi.eva.accession.release.batch.io.deprecated; -import com.mongodb.MongoClient; +import com.mongodb.client.MongoClient; import com.mongodb.client.model.Aggregates; import com.mongodb.client.model.Field; import com.mongodb.client.model.Filters; @@ -67,33 +67,33 @@ private String getInactiveField(String field) { @Override protected List buildAggregation() { Bson matchAssembly = Aggregates.match(Filters.eq(getInactiveField(REFERENCE_ASSEMBLY_FIELD), - assemblyAccession)); + assemblyAccession)); Bson matchMerged = Aggregates.match(Filters.eq(EVENT_TYPE_FIELD, EventType.DEPRECATED.toString())); Bson sort = Aggregates.sort(orderBy(ascending(getInactiveField(CONTIG_FIELD), getInactiveField(START_FIELD)))); List aggregation = new ArrayList<>(Arrays.asList(matchAssembly, matchMerged, sort)); for (String submittedVariantOperationCollectionName : allSubmittedVariantOperationCollectionNames) { Bson lookup = Aggregates.lookup(submittedVariantOperationCollectionName, ACCESSION_FIELD, - getInactiveField(CLUSTERED_VARIANT_ACCESSION_FIELD), - submittedVariantOperationCollectionName); + getInactiveField(CLUSTERED_VARIANT_ACCESSION_FIELD), + submittedVariantOperationCollectionName); aggregation.add(lookup); } // Concat entries from all submitted variant operation collections Bson ssConcat = Aggregates.addFields(new Field<>(SS_INFO_FIELD, - new Document("$concatArrays", allSubmittedVariantOperationCollectionNames - .stream().map(v -> "$" + v) - .collect(Collectors.toList())))); + new Document("$concatArrays", allSubmittedVariantOperationCollectionNames + .stream().map(v -> "$" + v) + .collect(Collectors.toList())))); aggregation.add(ssConcat); // Ensure that we are only retrieving the variants with the relevant taxonomy // and event type in the Submitted operations collections Bson matchTaxonomyAndEventType = Aggregates.match(Filters.and( Filters.ne(SS_INFO_FIELD, Collections.emptyList()), Filters.eq(SS_INFO_FIELD + "." + - getInactiveField(REFERENCE_ASSEMBLY_FIELD_IN_SUBMITTED_COLLECTIONS), - this.assemblyAccession), + getInactiveField(REFERENCE_ASSEMBLY_FIELD_IN_SUBMITTED_COLLECTIONS), + this.assemblyAccession), Filters.eq(SS_INFO_FIELD + "." + getInactiveField(TAXONOMY_FIELD), this.taxonomyAccession), Filters.in(SS_INFO_FIELD + "." + EVENT_TYPE_FIELD, - Arrays.asList(EventType.UPDATED.toString(), EventType.DEPRECATED.toString())))); + Arrays.asList(EventType.UPDATED.toString(), EventType.DEPRECATED.toString())))); aggregation.add(matchTaxonomyAndEventType); logger.info("Issuing aggregation: {}", aggregation); logger.info(aggregation.toString()); diff --git a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/merged/MergedVariantContextWriter.java b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/merged/MergedVariantContextWriter.java index 93712ab4b..fbbf2d77d 100644 --- a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/merged/MergedVariantContextWriter.java +++ b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/merged/MergedVariantContextWriter.java @@ -18,9 +18,8 @@ import htsjdk.variant.vcf.VCFHeaderLine; import htsjdk.variant.vcf.VCFHeaderLineType; import htsjdk.variant.vcf.VCFInfoHeaderLine; - -import uk.ac.ebi.eva.accession.release.batch.io.active.VariantContextWriter; import uk.ac.ebi.eva.accession.release.batch.io.VariantMongoAggregationReader; +import uk.ac.ebi.eva.accession.release.batch.io.active.VariantContextWriter; import java.nio.file.Path; import java.util.Set; diff --git a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/merged/MergedVariantMongoReader.java b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/merged/MergedVariantMongoReader.java index 6a43bdcc0..8daa07389 100644 --- a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/merged/MergedVariantMongoReader.java +++ b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/merged/MergedVariantMongoReader.java @@ -16,7 +16,7 @@ package uk.ac.ebi.eva.accession.release.batch.io.merged; -import com.mongodb.MongoClient; +import com.mongodb.client.MongoClient; import com.mongodb.client.model.Aggregates; import com.mongodb.client.model.Field; import com.mongodb.client.model.Filters; @@ -94,44 +94,44 @@ public void open(ExecutionContext executionContext) throws ItemStreamException { @Override protected List buildAggregation() { Bson matchAssembly = Aggregates.match(Filters.eq(getInactiveField(REFERENCE_ASSEMBLY_FIELD), - assemblyAccession)); + assemblyAccession)); Bson matchMerged = Aggregates.match(Filters.eq(EVENT_TYPE_FIELD, EventType.MERGED.toString())); Bson sort = Aggregates.sort(orderBy(ascending(getInactiveField(CONTIG_FIELD), getInactiveField(START_FIELD)))); List aggregation = new ArrayList<>(Arrays.asList(matchAssembly, matchMerged, sort)); for (String submittedVariantOperationCollectionName : allSubmittedVariantOperationCollectionNames) { Bson lookup = Aggregates.lookup(submittedVariantOperationCollectionName, ACCESSION_FIELD, - getInactiveField(CLUSTERED_VARIANT_ACCESSION_FIELD), - submittedVariantOperationCollectionName); + getInactiveField(CLUSTERED_VARIANT_ACCESSION_FIELD), + submittedVariantOperationCollectionName); aggregation.add(lookup); } // Concat entries from all submitted variant operation collections Bson ssConcat = Aggregates.addFields(new Field<>(SS_INFO_FIELD, - new Document("$concatArrays", allSubmittedVariantOperationCollectionNames - .stream().map(v -> "$" + v) - .collect(Collectors.toList())))); + new Document("$concatArrays", allSubmittedVariantOperationCollectionNames + .stream().map(v -> "$" + v) + .collect(Collectors.toList())))); aggregation.add(ssConcat); // Ensure that we are only retrieving the variants with the relevant taxonomy // and event type in the Submitted operations collections Bson matchTaxonomyAndEventType = Aggregates.match(Filters.and( Filters.ne(SS_INFO_FIELD, Collections.emptyList()), Filters.eq(SS_INFO_FIELD + "." + - getInactiveField(REFERENCE_ASSEMBLY_FIELD_IN_SUBMITTED_COLLECTIONS), - this.assemblyAccession), + getInactiveField(REFERENCE_ASSEMBLY_FIELD_IN_SUBMITTED_COLLECTIONS), + this.assemblyAccession), Filters.eq(SS_INFO_FIELD + "." + getInactiveField(TAXONOMY_FIELD), this.taxonomyAccession), - Filters.eq(SS_INFO_FIELD + "." + EVENT_TYPE_FIELD,EventType.UPDATED.toString()))); + Filters.eq(SS_INFO_FIELD + "." + EVENT_TYPE_FIELD, EventType.UPDATED.toString()))); aggregation.add(matchTaxonomyAndEventType); // Similarly look in both clustered variant collections for active RS for (String clusteredVariantCollectionName : allClusteredVariantCollectionNames) { Bson lookupClusteredVariants = Aggregates.lookup(clusteredVariantCollectionName, MERGE_INTO_FIELD, - ACCESSION_FIELD, clusteredVariantCollectionName); + ACCESSION_FIELD, clusteredVariantCollectionName); aggregation.add(lookupClusteredVariants); } Bson rsConcat = Aggregates.addFields(new Field<>(ACTIVE_RS, - new Document("$concatArrays", allClusteredVariantCollectionNames - .stream().map(v -> "$" + v) - .collect(Collectors.toList())))); + new Document("$concatArrays", allClusteredVariantCollectionNames + .stream().map(v -> "$" + v) + .collect(Collectors.toList())))); aggregation.add(rsConcat); Bson matchOnlyNonEmptyActiveRS = Aggregates.match(Filters.ne(ACTIVE_RS, Collections.emptyList())); aggregation.add(matchOnlyNonEmptyActiveRS); @@ -148,9 +148,9 @@ protected List getVariants(Document mergedVariant) { Collection inactiveObjects = (Collection) mergedVariant.get(INACTIVE_OBJECTS); if (inactiveObjects.size() > 1) { throw new AssertionError("The class '" + this.getClass().getSimpleName() - + "' was designed assuming there's only one element in the field " - + "'" + INACTIVE_OBJECTS + "'. Found " + inactiveObjects.size() - + " elements in _id=" + mergedVariant.get(ACCESSION_FIELD)); + + "' was designed assuming there's only one element in the field " + + "'" + INACTIVE_OBJECTS + "'. Found " + inactiveObjects.size() + + " elements in _id=" + mergedVariant.get(ACCESSION_FIELD)); } Document inactiveEntity = inactiveObjects.iterator().next(); String contig = inactiveEntity.getString(VariantMongoAggregationReader.CONTIG_FIELD); @@ -164,9 +164,9 @@ protected List getVariants(Document mergedVariant) { Map mergedVariants = new HashMap<>(); Collection submittedVariantOperations = (Collection) mergedVariant.get(SS_INFO_FIELD); boolean remappedRS = submittedVariantOperations.stream() - .map(e -> (Collection) e.get("inactiveObjects")) - .flatMap(Collection::stream) - .allMatch(sve -> Objects.nonNull(sve.getString("remappedFrom"))); + .map(e -> (Collection) e.get("inactiveObjects")) + .flatMap(Collection::stream) + .allMatch(sve -> Objects.nonNull(sve.getString("remappedFrom"))); Collection activeClusteredVariant = (Collection) mergedVariant.get(ACTIVE_RS); if (activeClusteredVariant.isEmpty()) { @@ -183,7 +183,7 @@ protected List getVariants(Document mergedVariant) { long submittedVariantStart = submittedVariant.getLong(START_FIELD); String submittedVariantContig = submittedVariant.getString(CONTIG_FIELD); - if (!isSameLocation(contig, start, submittedVariantContig, submittedVariantStart, type)){ + if (!isSameLocation(contig, start, submittedVariantContig, submittedVariantStart, type)) { continue; } @@ -197,9 +197,9 @@ protected List getVariants(Document mergedVariant) { .getBoolean(SUPPORTED_BY_EVIDENCE_FIELD, DEFAULT_SUPPORTED_BY_EVIDENCE); VariantSourceEntry sourceEntry = buildVariantSourceEntry(study, sequenceOntology, validated, - submittedVariantValidated, allelesMatch, - assemblyMatch, evidence, remappedRS, - mergedInto); + submittedVariantValidated, allelesMatch, + assemblyMatch, evidence, remappedRS, + mergedInto); addToVariants(mergedVariants, contig, submittedVariantStart, rs, reference, alternate, sourceEntry); } @@ -212,10 +212,10 @@ protected List getVariants(Document mergedVariant) { if (!hasSubmittedVariantsDeclustered && mergedVariants.isEmpty()) { logger.warn("Found merge operation for rs" + rs + " but no SS IDs updates " + - "(merge/update) in the collection containing operations. " + - "This could have possibly happened on a remapped variant " + - "because there was a subsequent split issued for this RS due to loci disagreement " + - "between the RS and the SS."); + "(merge/update) in the collection containing operations. " + + "This could have possibly happened on a remapped variant " + + "because there was a subsequent split issued for this RS due to loci disagreement " + + "between the RS and the SS."); return Collections.emptyList(); } diff --git a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/merged_deprecated/DbsnpMergedDeprecatedVariantMongoReader.java b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/merged_deprecated/DbsnpMergedDeprecatedVariantMongoReader.java index 071ab00ab..ffd885b98 100644 --- a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/merged_deprecated/DbsnpMergedDeprecatedVariantMongoReader.java +++ b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/merged_deprecated/DbsnpMergedDeprecatedVariantMongoReader.java @@ -17,7 +17,7 @@ package uk.ac.ebi.eva.accession.release.batch.io.merged_deprecated; import com.mongodb.BasicDBObject; -import com.mongodb.MongoClient; +import com.mongodb.client.MongoClient; import org.bson.Document; import org.springframework.data.mongodb.core.convert.MongoConverter; @@ -55,10 +55,10 @@ public DbsnpMergedDeprecatedVariantMongoReader(String assemblyAccession, int tax protected DbsnpClusteredVariantOperationEntity getEntity(Document operation) { List objects = (List) operation.get(INACTIVE_OBJECTS); operation.put(INACTIVE_OBJECTS, objects.stream() - .map(BasicDBObject::new) - .map(o -> mongoConverter.read(DbsnpClusteredVariantInactiveEntity.class, - o)) - .collect(Collectors.toList())); + .map(BasicDBObject::new) + .map(o -> mongoConverter.read(DbsnpClusteredVariantInactiveEntity.class, + o)) + .collect(Collectors.toList())); return mongoConverter.read(DbsnpClusteredVariantOperationEntity.class, new BasicDBObject(operation)); } diff --git a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/merged_deprecated/EvaMergedDeprecatedVariantMongoReader.java b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/merged_deprecated/EvaMergedDeprecatedVariantMongoReader.java index 8fc151ad4..de078399c 100644 --- a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/merged_deprecated/EvaMergedDeprecatedVariantMongoReader.java +++ b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/merged_deprecated/EvaMergedDeprecatedVariantMongoReader.java @@ -17,7 +17,7 @@ package uk.ac.ebi.eva.accession.release.batch.io.merged_deprecated; import com.mongodb.BasicDBObject; -import com.mongodb.MongoClient; +import com.mongodb.client.MongoClient; import org.bson.Document; import org.springframework.data.mongodb.core.convert.MongoConverter; import uk.ac.ebi.ampt2d.commons.accession.persistence.mongodb.document.EventDocument; @@ -56,10 +56,10 @@ public EvaMergedDeprecatedVariantMongoReader(String assemblyAccession, int taxon protected ClusteredVariantOperationEntity getEntity(Document operation) { List objects = (List) operation.get(INACTIVE_OBJECTS); operation.put(INACTIVE_OBJECTS, objects.stream() - .map(BasicDBObject::new) - .map(o -> mongoConverter.read(ClusteredVariantInactiveEntity.class, - o)) - .collect(Collectors.toList())); + .map(BasicDBObject::new) + .map(o -> mongoConverter.read(ClusteredVariantInactiveEntity.class, + o)) + .collect(Collectors.toList())); return mongoConverter.read(ClusteredVariantOperationEntity.class, new BasicDBObject(operation)); } diff --git a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/merged_deprecated/MergedDeprecatedVariantAccessionWriter.java b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/merged_deprecated/MergedDeprecatedVariantAccessionWriter.java index 3e6a9ce06..4577ce040 100644 --- a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/merged_deprecated/MergedDeprecatedVariantAccessionWriter.java +++ b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/merged_deprecated/MergedDeprecatedVariantAccessionWriter.java @@ -15,6 +15,7 @@ */ package uk.ac.ebi.eva.accession.release.batch.io.merged_deprecated; +import org.springframework.batch.item.Chunk; import org.springframework.batch.item.ExecutionContext; import org.springframework.batch.item.ItemStreamException; import org.springframework.batch.item.ItemStreamWriter; @@ -27,7 +28,6 @@ import java.io.IOException; import java.io.PrintWriter; import java.nio.file.Path; -import java.util.List; /** * Writes the accessions of historical variants, that have been merged into a later deprecate one, to a flat file. @@ -57,7 +57,7 @@ public void open(ExecutionContext executionContext) throws ItemStreamException { } @Override - public void write(List>> variants) throws Exception { for (EventDocument> variant : variants) { diff --git a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/merged_deprecated/MergedDeprecatedVariantMongoReader.java b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/merged_deprecated/MergedDeprecatedVariantMongoReader.java index db5522712..270941a31 100644 --- a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/merged_deprecated/MergedDeprecatedVariantMongoReader.java +++ b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/merged_deprecated/MergedDeprecatedVariantMongoReader.java @@ -16,8 +16,8 @@ package uk.ac.ebi.eva.accession.release.batch.io.merged_deprecated; -import com.mongodb.MongoClient; import com.mongodb.client.AggregateIterable; +import com.mongodb.client.MongoClient; import com.mongodb.client.MongoCollection; import com.mongodb.client.MongoCursor; import com.mongodb.client.MongoDatabase; @@ -31,11 +31,7 @@ import org.springframework.batch.item.ExecutionContext; import org.springframework.batch.item.ItemStreamException; import org.springframework.batch.item.ItemStreamReader; -import org.springframework.batch.item.NonTransientResourceException; -import org.springframework.batch.item.ParseException; -import org.springframework.batch.item.UnexpectedInputException; import uk.ac.ebi.ampt2d.commons.accession.core.models.EventType; - import uk.ac.ebi.eva.accession.release.collectionNames.CollectionNames; import java.util.ArrayList; @@ -110,9 +106,8 @@ private void aggregate(String collectionName) { MongoDatabase db = mongoClient.getDatabase(database); MongoCollection collection = db.getCollection(collectionName); AggregateIterable clusteredVariants = collection.aggregate(buildAggregation()) - .allowDiskUse(true) - .useCursor(true) - .batchSize(chunkSize); + .allowDiskUse(true) + .batchSize(chunkSize); cursor = clusteredVariants.iterator(); } @@ -121,32 +116,32 @@ private List buildAggregation() { Bson matchAssembly = Aggregates.match(Filters.eq(getInactiveField(ASSEMBLY_FIELD), assemblyAccession)); Bson matchMerged = Aggregates.match(Filters.eq(EVENT_TYPE_FIELD, EventType.MERGED.toString())); Bson lookup = Aggregates.lookup(names.getSubmittedVariantEntity(), MERGE_INTO_FIELD, - CLUSTERED_VARIANT_ACCESSION_FIELD, SS_INFO_FIELD); + CLUSTERED_VARIANT_ACCESSION_FIELD, SS_INFO_FIELD); Bson matchEmpty = Aggregates.match(Filters.eq(SS_INFO_FIELD, Collections.EMPTY_LIST)); aggregation.addAll(Arrays.asList(matchAssembly, matchMerged, lookup, matchEmpty)); for (String submittedVariantOperationCollectionName : allSubmittedVariantOperationCollectionNames) { Bson opsLookup = Aggregates.lookup(submittedVariantOperationCollectionName, ACCESSION_FIELD, - getInactiveField(CLUSTERED_VARIANT_ACCESSION_FIELD), - submittedVariantOperationCollectionName); + getInactiveField(CLUSTERED_VARIANT_ACCESSION_FIELD), + submittedVariantOperationCollectionName); aggregation.add(opsLookup); } // Concat entries from all submitted variant operation collections Bson ssOpsConcat = Aggregates.addFields(new Field<>(SS_OPS_INFO_FIELD, - new Document("$concatArrays", - allSubmittedVariantOperationCollectionNames - .stream().map(v -> "$" + v) - .collect(Collectors.toList())))); + new Document("$concatArrays", + allSubmittedVariantOperationCollectionNames + .stream().map(v -> "$" + v) + .collect(Collectors.toList())))); aggregation.add(ssOpsConcat); // There should be some evidence that there is at least one operation in the submitted operations // referencing the said RS Bson matchAtLeastOneSSOp = Aggregates.match(Filters.and( Filters.ne(SS_OPS_INFO_FIELD, Collections.emptyList()), Filters.eq(SS_OPS_INFO_FIELD + "." + - getInactiveField(TAXONOMY_FIELD), - this.taxonomyAccession), + getInactiveField(TAXONOMY_FIELD), + this.taxonomyAccession), Filters.eq(SS_OPS_INFO_FIELD + "." + - getInactiveField(REFERENCE_ASSEMBLY_FIELD_IN_SUBMITTED_COLLECTIONS), - this.assemblyAccession))); + getInactiveField(REFERENCE_ASSEMBLY_FIELD_IN_SUBMITTED_COLLECTIONS), + this.assemblyAccession))); aggregation.add(matchAtLeastOneSSOp); logger.info("Issuing aggregation: {}", aggregation); return aggregation; @@ -157,8 +152,7 @@ private String getInactiveField(String field) { } @Override - public OPERATION_ENTITY read() - throws Exception, UnexpectedInputException, ParseException, NonTransientResourceException { + public OPERATION_ENTITY read() throws Exception { return cursor.hasNext() ? getEntity(cursor.next()) : null; } diff --git a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/multimap/MultimapVariantContextWriter.java b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/multimap/MultimapVariantContextWriter.java index 9693ab369..19ab2bd30 100644 --- a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/multimap/MultimapVariantContextWriter.java +++ b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/multimap/MultimapVariantContextWriter.java @@ -39,8 +39,8 @@ public MultimapVariantContextWriter(Path outputPath, String referenceAssembly, S protected Set buildHeaderLines() { Set vcfHeaderLines = super.buildHeaderLines(); vcfHeaderLines.add(new VCFInfoHeaderLine(MAPPING_WEIGHT_KEY, 1, VCFHeaderLineType.Integer, - "mapping weight as defined by dbSNP for database tables at " + - DBSNP_MAP_WEIGHT_DEFINITION_URL)); + "mapping weight as defined by dbSNP for database tables at " + + DBSNP_MAP_WEIGHT_DEFINITION_URL)); return vcfHeaderLines; } diff --git a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/multimap/MultimapVariantMongoReader.java b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/multimap/MultimapVariantMongoReader.java index a7f7446f1..647df8433 100644 --- a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/multimap/MultimapVariantMongoReader.java +++ b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/io/multimap/MultimapVariantMongoReader.java @@ -15,7 +15,7 @@ */ package uk.ac.ebi.eva.accession.release.batch.io.multimap; -import com.mongodb.MongoClient; +import com.mongodb.client.MongoClient; import com.mongodb.client.model.Aggregates; import com.mongodb.client.model.Filters; import org.bson.Document; @@ -48,11 +48,11 @@ public MultimapVariantMongoReader(String assemblyAccession, int taxonomyAccessio @Override protected List buildAggregation() { Bson match = Aggregates.match(Filters.and(Filters.eq(REFERENCE_ASSEMBLY_FIELD, assemblyAccession), - Filters.eq(TAXONOMY_FIELD, taxonomyAccession), - Filters.gte(MAPPING_WEIGHT_FIELD, NON_SINGLE_LOCATION_MAPPING))); + Filters.eq(TAXONOMY_FIELD, taxonomyAccession), + Filters.gte(MAPPING_WEIGHT_FIELD, NON_SINGLE_LOCATION_MAPPING))); Bson sort = Aggregates.sort(orderBy(ascending(CONTIG_FIELD, START_FIELD))); Bson lookup = Aggregates.lookup(names.getSubmittedVariantEntity(), ACCESSION_FIELD, - CLUSTERED_VARIANT_ACCESSION_FIELD, SS_INFO_FIELD); + CLUSTERED_VARIANT_ACCESSION_FIELD, SS_INFO_FIELD); List aggregation = Arrays.asList(match, sort, lookup); logger.info("Issuing aggregation: {}", aggregation); return aggregation; @@ -63,7 +63,7 @@ protected List getVariants(Document clusteredVariant) { List variants = super.getVariants(clusteredVariant); for (Variant variant : variants) { variant.getSourceEntries().iterator().next().addAttribute(MAPPING_WEIGHT_KEY, - clusteredVariant.get(MAPPING_WEIGHT_FIELD).toString()); + clusteredVariant.get(MAPPING_WEIGHT_FIELD).toString()); } return variants; } diff --git a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/listeners/ExcludeVariantsListener.java b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/listeners/ExcludeVariantsListener.java index 075e590cb..3446ac767 100644 --- a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/listeners/ExcludeVariantsListener.java +++ b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/listeners/ExcludeVariantsListener.java @@ -32,7 +32,7 @@ public class ExcludeVariantsListener extends StepListenerSupport * VCF format only accepts reference and alternate alleles formed by A, C, G, T or N letters in upper or lower case. * If one of the alleles has a different character, this processor will return null so that variant can be ignored. */ @@ -40,9 +40,6 @@ public class ExcludeInvalidVariantsProcessor implements ItemProcessor * Examples of structural variants taken from * VCFv.3 spec (section 5.3): * 

@@ -47,9 +47,9 @@
  * 3       9425916 .    C      23   PASS   SVTYPE=INS;END=9425916;SVLEN=6027
  * }
  *
- * + *

* Note that unlike regular INDELS, variants with symbolic alleles have the context bases only in the REF column. - * + *

* Also, note that the deletions have the symbolic allele in the ALT column. */ public class NamedVariantProcessor implements ItemProcessor { @@ -58,9 +58,9 @@ public class NamedVariantProcessor implements ItemProcessor { private static Map sequenceOntologyToVariantType = Arrays.stream(VariantType.values()) - .filter(type -> type != VariantType.NO_ALTERNATE) - .collect(Collectors.toMap(VariantTypeToSOAccessionMap::getSequenceOntologyAccession, - type -> type)); + .filter(type -> type != VariantType.NO_ALTERNATE) + .collect(Collectors.toMap(VariantTypeToSOAccessionMap::getSequenceOntologyAccession, + type -> type)); @Override public IVariant process(Variant variant) throws Exception { @@ -75,7 +75,7 @@ public IVariant process(Variant variant) throws Exception { if (isNamedAllele(oldAlternate) || isSymbolicAllele(oldAlternate)) { throw new IllegalArgumentException( "This variant (with named/symbolic alleles in both the reference and alternate alleles) can't" - + " be written in VCF, as only the ALT column can have symbolic alleles: " + variant); + + " be written in VCF, as only the ALT column can have symbolic alleles: " + variant); } else { // swap the alleles, look this class' documentation newReference = oldAlternate; @@ -86,7 +86,7 @@ public IVariant process(Variant variant) throws Exception { } Variant newVariant = new Variant(variant.getChromosome(), variant.getStart(), variant.getEnd(), newReference, - makeAltAlleleValid(newAlternate)); + makeAltAlleleValid(newAlternate)); newVariant.addSourceEntries(variant.getSourceEntries()); newVariant.setMainId(variant.getMainId()); @@ -131,19 +131,19 @@ private void logIfVariantHasWrongType(Variant variant) { boolean isSequenceAlteration = variant.getType() == VariantType.SEQUENCE_ALTERATION; if (isSequenceAlteration && !containsSequenceAlterationInMongo - || !isSequenceAlteration && containsSequenceAlterationInMongo) { + || !isSequenceAlteration && containsSequenceAlterationInMongo) { logger.warn("Variant is stored in MongoDB with types " + types + ", but AbstractVariant::getType says" - + " that the correct type is " + variant.getType() + ". The variant (" + variant.getMainId() - + ") is: " + variant); + + " that the correct type is " + variant.getType() + ". The variant (" + variant.getMainId() + + ") is: " + variant); } } private Set getTypeFromAttributes(Variant variant) { return variant.getSourceEntries() - .stream() - .map(AbstractVariantSourceEntry::getAttributes) - .map(attributes -> attributes.get(VARIANT_CLASS_KEY)) - .map(sequenceOntologyToVariantType::get) - .collect(Collectors.toSet()); + .stream() + .map(AbstractVariantSourceEntry::getAttributes) + .map(attributes -> attributes.get(VARIANT_CLASS_KEY)) + .map(sequenceOntologyToVariantType::get) + .collect(Collectors.toSet()); } } diff --git a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/processors/VariantToVariantContextProcessor.java b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/processors/VariantToVariantContextProcessor.java index 4d3e409a5..6b4803048 100644 --- a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/processors/VariantToVariantContextProcessor.java +++ b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/batch/processors/VariantToVariantContextProcessor.java @@ -19,7 +19,6 @@ import htsjdk.variant.variantcontext.VariantContext; import htsjdk.variant.variantcontext.VariantContextBuilder; import org.springframework.batch.item.ItemProcessor; - import uk.ac.ebi.eva.accession.core.contig.ContigMapping; import uk.ac.ebi.eva.accession.core.contig.ContigNaming; import uk.ac.ebi.eva.accession.core.contig.ContigSynonyms; @@ -33,6 +32,7 @@ import java.util.Map; import java.util.stream.Collectors; +import static uk.ac.ebi.eva.accession.release.batch.io.VariantMongoAggregationReader.MAPPING_WEIGHT_KEY; import static uk.ac.ebi.eva.accession.release.batch.io.active.AccessionedVariantMongoReader.ALLELES_MATCH_KEY; import static uk.ac.ebi.eva.accession.release.batch.io.active.AccessionedVariantMongoReader.ASSEMBLY_MATCH_KEY; import static uk.ac.ebi.eva.accession.release.batch.io.active.AccessionedVariantMongoReader.CLUSTERED_VARIANT_VALIDATED_KEY; @@ -42,11 +42,10 @@ import static uk.ac.ebi.eva.accession.release.batch.io.active.AccessionedVariantMongoReader.SUPPORTED_BY_EVIDENCE_KEY; import static uk.ac.ebi.eva.accession.release.batch.io.active.AccessionedVariantMongoReader.VARIANT_CLASS_KEY; import static uk.ac.ebi.eva.accession.release.batch.io.merged.MergedVariantMongoReader.MERGED_INTO_KEY; -import static uk.ac.ebi.eva.accession.release.batch.io.VariantMongoAggregationReader.MAPPING_WEIGHT_KEY; /** * Converts an IVariant to a VariantContext. - * + *

* The latter can be serialized using HTSJDK. This processor requires any to-be-serialized property to be set in the * IVariant already, such as the alleles. */ @@ -103,7 +102,7 @@ private Map getAttributes(IVariant variant) { case MERGED_INTO_KEY: case MAPPING_WEIGHT_KEY: attributes.put(attribute.getKey(), - toUniqueConcatenation(replaceInvalidCharacters(attribute.getValue()))); + toUniqueConcatenation(replaceInvalidCharacters(attribute.getValue()))); break; case ALLELES_MATCH_KEY: case ASSEMBLY_MATCH_KEY: @@ -141,8 +140,8 @@ private Map getAttributes(IVariant variant) { */ private List replaceInvalidCharacters(List infoValues) { return infoValues.stream() - .map(s -> s.replaceAll("[ ,;=]", "_")) - .collect(Collectors.toList()); + .map(s -> s.replaceAll("[ ,;=]", "_")) + .collect(Collectors.toList()); } private String toUniqueConcatenation(List value) { diff --git a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/RunnerConfiguration.java b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/RunnerConfiguration.java index a9792312b..a0c05b893 100644 --- a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/RunnerConfiguration.java +++ b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/RunnerConfiguration.java @@ -16,12 +16,6 @@ */ package uk.ac.ebi.eva.accession.release.configuration; -import org.springframework.batch.core.configuration.annotation.BatchConfigurer; -import org.springframework.batch.core.configuration.annotation.DefaultBatchConfigurer; -import org.springframework.batch.core.explore.JobExplorer; -import org.springframework.batch.core.explore.support.MapJobExplorerFactoryBean; -import org.springframework.batch.core.repository.JobRepository; -import org.springframework.batch.core.repository.support.MapJobRepositoryFactoryBean; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import uk.ac.ebi.eva.commons.batch.job.JobExecutionApplicationListener; @@ -33,19 +27,4 @@ public class RunnerConfiguration { public JobExecutionApplicationListener jobExecutionApplicationListener() { return new JobExecutionApplicationListener(); } - - @Bean - public BatchConfigurer configurer() { - return new DefaultBatchConfigurer() { - @Override - protected JobRepository createJobRepository() throws Exception { - return new MapJobRepositoryFactoryBean().getObject(); - } - - @Override - protected JobExplorer createJobExplorer() throws Exception { - return new MapJobExplorerFactoryBean((MapJobRepositoryFactoryBean) new MapJobRepositoryFactoryBean()).getObject(); - } - }; - } } \ No newline at end of file diff --git a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/jobs/ActiveAccessionReleaseFromDBJobConfiguration.java b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/jobs/ActiveAccessionReleaseFromDBJobConfiguration.java index 7c8e522fb..fe595823c 100644 --- a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/jobs/ActiveAccessionReleaseFromDBJobConfiguration.java +++ b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/jobs/ActiveAccessionReleaseFromDBJobConfiguration.java @@ -3,8 +3,9 @@ import org.springframework.batch.core.Job; import org.springframework.batch.core.Step; import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; -import org.springframework.batch.core.configuration.annotation.JobBuilderFactory; +import org.springframework.batch.core.job.builder.JobBuilder; import org.springframework.batch.core.launch.support.RunIdIncrementer; +import org.springframework.batch.core.repository.JobRepository; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; @@ -22,8 +23,8 @@ public class ActiveAccessionReleaseFromDBJobConfiguration { private Step activeAccessionsReleaseFromDBStep; @Bean(ACTIVE_ACCESSIONS_RELEASE_FROM_DB_JOB) - public Job activeAccessionReleaseFromDBJob(JobBuilderFactory jobBuilderFactory) { - return jobBuilderFactory.get(ACTIVE_ACCESSIONS_RELEASE_FROM_DB_JOB) + public Job activeAccessionReleaseFromDBJob(JobRepository jobRepository) { + return new JobBuilder(ACTIVE_ACCESSIONS_RELEASE_FROM_DB_JOB, jobRepository) .incrementer(new RunIdIncrementer()) .start(activeAccessionsReleaseFromDBStep) .build(); diff --git a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/jobs/DumpRSAccessionsJobConfiguration.java b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/jobs/DumpRSAccessionsJobConfiguration.java index 9b7772816..e8be4c938 100644 --- a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/jobs/DumpRSAccessionsJobConfiguration.java +++ b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/jobs/DumpRSAccessionsJobConfiguration.java @@ -4,8 +4,9 @@ import org.springframework.batch.core.Job; import org.springframework.batch.core.Step; import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; -import org.springframework.batch.core.configuration.annotation.JobBuilderFactory; +import org.springframework.batch.core.job.builder.JobBuilder; import org.springframework.batch.core.launch.support.RunIdIncrementer; +import org.springframework.batch.core.repository.JobRepository; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; @@ -29,16 +30,16 @@ public class DumpRSAccessionsJobConfiguration { private Step dumpMergedAndDeprecatedAccessionsStep; @Bean(DUMP_ACTIVE_ACCESSIONS_JOB) - public Job dumpActiveAccessionJob(JobBuilderFactory jobBuilderFactory) { - return jobBuilderFactory.get(DUMP_ACTIVE_ACCESSIONS_JOB) + public Job dumpActiveAccessionJob(JobRepository jobRepository) { + return new JobBuilder(DUMP_ACTIVE_ACCESSIONS_JOB, jobRepository) .incrementer(new RunIdIncrementer()) .start(dumpActiveAccessionsStep) .build(); } @Bean(DUMP_MERGED_AND_DEPRECATED_ACCESSIONS_JOB) - public Job dumpMergedAndDeprecatedAccessionJob(JobBuilderFactory jobBuilderFactory) { - return jobBuilderFactory.get(DUMP_MERGED_AND_DEPRECATED_ACCESSIONS_JOB) + public Job dumpMergedAndDeprecatedAccessionJob(JobRepository jobRepository) { + return new JobBuilder(DUMP_MERGED_AND_DEPRECATED_ACCESSIONS_JOB, jobRepository) .incrementer(new RunIdIncrementer()) .start(dumpMergedAndDeprecatedAccessionsStep) .build(); diff --git a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/jobs/MergedAndDeprecatedAccessionReleaseFromDBJobConfiguration.java b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/jobs/MergedAndDeprecatedAccessionReleaseFromDBJobConfiguration.java index e442f067b..acf2c36b6 100644 --- a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/jobs/MergedAndDeprecatedAccessionReleaseFromDBJobConfiguration.java +++ b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/jobs/MergedAndDeprecatedAccessionReleaseFromDBJobConfiguration.java @@ -3,8 +3,9 @@ import org.springframework.batch.core.Job; import org.springframework.batch.core.Step; import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; -import org.springframework.batch.core.configuration.annotation.JobBuilderFactory; +import org.springframework.batch.core.job.builder.JobBuilder; import org.springframework.batch.core.launch.support.RunIdIncrementer; +import org.springframework.batch.core.repository.JobRepository; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; @@ -22,8 +23,8 @@ public class MergedAndDeprecatedAccessionReleaseFromDBJobConfiguration { private Step mergedAndDeprecatedAccessionsReleaseFromDBStep; @Bean(MERGED_AND_DEPRECATED_ACCESSIONS_RELEASE_FROM_DB_JOB) - public Job mergedAndDeprecatedAccessionReleaseFromDBJob(JobBuilderFactory jobBuilderFactory) { - return jobBuilderFactory.get(MERGED_AND_DEPRECATED_ACCESSIONS_RELEASE_FROM_DB_JOB) + public Job mergedAndDeprecatedAccessionReleaseFromDBJob(JobRepository jobRepository) { + return new JobBuilder(MERGED_AND_DEPRECATED_ACCESSIONS_RELEASE_FROM_DB_JOB, jobRepository) .incrementer(new RunIdIncrementer()) .start(mergedAndDeprecatedAccessionsReleaseFromDBStep) .build(); diff --git a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/processors/ReleaseProcessorConfiguration.java b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/processors/ReleaseProcessorConfiguration.java index ff0423234..acf161c86 100644 --- a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/processors/ReleaseProcessorConfiguration.java +++ b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/processors/ReleaseProcessorConfiguration.java @@ -21,14 +21,13 @@ import org.springframework.batch.item.support.CompositeItemProcessor; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; - -import uk.ac.ebi.eva.accession.core.contig.ContigMapping; import uk.ac.ebi.eva.accession.core.batch.io.FastaSynonymSequenceReader; -import uk.ac.ebi.eva.accession.release.parameters.InputParameters; +import uk.ac.ebi.eva.accession.core.contig.ContigMapping; import uk.ac.ebi.eva.accession.release.batch.processors.ContextNucleotideAdditionProcessor; import uk.ac.ebi.eva.accession.release.batch.processors.ExcludeInvalidVariantsProcessor; import uk.ac.ebi.eva.accession.release.batch.processors.NamedVariantProcessor; import uk.ac.ebi.eva.accession.release.batch.processors.VariantToVariantContextProcessor; +import uk.ac.ebi.eva.accession.release.parameters.InputParameters; import uk.ac.ebi.eva.commons.core.models.pipeline.Variant; import java.io.IOException; diff --git a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/steps/ActiveAccessionReleaseFromDBStepConfiguration.java b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/steps/ActiveAccessionReleaseFromDBStepConfiguration.java index ba9a87d33..be3a1e5d5 100644 --- a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/steps/ActiveAccessionReleaseFromDBStepConfiguration.java +++ b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/steps/ActiveAccessionReleaseFromDBStepConfiguration.java @@ -3,7 +3,8 @@ import htsjdk.variant.variantcontext.VariantContext; import org.springframework.batch.core.Step; import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; -import org.springframework.batch.core.configuration.annotation.StepBuilderFactory; +import org.springframework.batch.core.repository.JobRepository; +import org.springframework.batch.core.step.builder.StepBuilder; import org.springframework.batch.core.step.tasklet.TaskletStep; import org.springframework.batch.item.ItemProcessor; import org.springframework.batch.item.ItemReader; @@ -13,6 +14,7 @@ import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.transaction.PlatformTransactionManager; import uk.ac.ebi.eva.commons.core.models.pipeline.Variant; import static uk.ac.ebi.eva.accession.release.configuration.BeanNames.ACTIVE_ACCESSIONS_RELEASE_FROM_DB_STEP; @@ -37,10 +39,10 @@ public class ActiveAccessionReleaseFromDBStepConfiguration { ItemStreamWriter accessionWriter; @Bean(ACTIVE_ACCESSIONS_RELEASE_FROM_DB_STEP) - public Step activeAccessionsReleaseFromDBStep(StepBuilderFactory stepBuilderFactory, + public Step activeAccessionsReleaseFromDBStep(JobRepository jobRepository, PlatformTransactionManager transactionManager, SimpleCompletionPolicy chunkSizeCompletionPolicy) { - TaskletStep step = stepBuilderFactory.get(ACTIVE_ACCESSIONS_RELEASE_FROM_DB_STEP) - .chunk(chunkSizeCompletionPolicy) + TaskletStep step = new StepBuilder(ACTIVE_ACCESSIONS_RELEASE_FROM_DB_STEP, jobRepository) + .chunk(chunkSizeCompletionPolicy, transactionManager) .reader(variantReader) .processor(variantProcessor) .writer(accessionWriter) diff --git a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/steps/DumpRSAccessionsStepConfiguration.java b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/steps/DumpRSAccessionsStepConfiguration.java index abadaf279..762b22ac1 100644 --- a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/steps/DumpRSAccessionsStepConfiguration.java +++ b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/steps/DumpRSAccessionsStepConfiguration.java @@ -2,11 +2,13 @@ import org.springframework.batch.core.Step; import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; -import org.springframework.batch.core.configuration.annotation.StepBuilderFactory; +import org.springframework.batch.core.repository.JobRepository; +import org.springframework.batch.core.step.builder.StepBuilder; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.transaction.PlatformTransactionManager; import uk.ac.ebi.eva.accession.release.batch.io.DumpRSAccessionsInFile; import uk.ac.ebi.eva.accession.release.parameters.InputParameters; @@ -23,24 +25,26 @@ public class DumpRSAccessionsStepConfiguration { private DumpRSAccessionsInFile dumpRSAccessionsInFile; @Bean(DUMP_ACTIVE_ACCESSIONS_STEP) - public Step dumpActiveAccessionStep(StepBuilderFactory stepBuilderFactory, InputParameters inputParameters) { - return stepBuilderFactory.get(DUMP_ACTIVE_ACCESSIONS_STEP) + public Step dumpActiveAccessionStep(JobRepository jobRepository, PlatformTransactionManager transactionManager, + InputParameters inputParameters) { + return new StepBuilder(DUMP_ACTIVE_ACCESSIONS_STEP, jobRepository) .tasklet((contribution, chunkContext) -> { dumpRSAccessionsInFile.dumpAccessions(DumpRSAccessionsInFile.RSDumpType.ACTIVE, inputParameters.getAssemblyAccession()); return null; - }) + }, transactionManager) .build(); } @Bean(DUMP_MERGED_AND_DEPRECATED_ACCESSIONS_STEP) - public Step dumpMergedAndDeprecatedAccessionStep(StepBuilderFactory stepBuilderFactory, InputParameters inputParameters) { - return stepBuilderFactory.get(DUMP_MERGED_AND_DEPRECATED_ACCESSIONS_STEP) + public Step dumpMergedAndDeprecatedAccessionStep(JobRepository jobRepository, PlatformTransactionManager transactionManager, + InputParameters inputParameters) { + return new StepBuilder(DUMP_MERGED_AND_DEPRECATED_ACCESSIONS_STEP, jobRepository) .tasklet((contribution, chunkContext) -> { dumpRSAccessionsInFile.dumpAccessions(DumpRSAccessionsInFile.RSDumpType.MERGED_AND_DEPRECATED, inputParameters.getAssemblyAccession()); return null; - }) + }, transactionManager) .build(); } } \ No newline at end of file diff --git a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/steps/MergedAndDeprecatedAccessionReleaseFromDBStepConfiguration.java b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/steps/MergedAndDeprecatedAccessionReleaseFromDBStepConfiguration.java index 85f7893bc..a07221a2a 100644 --- a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/steps/MergedAndDeprecatedAccessionReleaseFromDBStepConfiguration.java +++ b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/steps/MergedAndDeprecatedAccessionReleaseFromDBStepConfiguration.java @@ -3,7 +3,8 @@ import htsjdk.variant.variantcontext.VariantContext; import org.springframework.batch.core.Step; import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; -import org.springframework.batch.core.configuration.annotation.StepBuilderFactory; +import org.springframework.batch.core.repository.JobRepository; +import org.springframework.batch.core.step.builder.StepBuilder; import org.springframework.batch.core.step.tasklet.TaskletStep; import org.springframework.batch.item.ItemProcessor; import org.springframework.batch.item.ItemReader; @@ -13,6 +14,7 @@ import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.transaction.PlatformTransactionManager; import uk.ac.ebi.eva.commons.core.models.pipeline.Variant; import static uk.ac.ebi.eva.accession.release.configuration.BeanNames.EVA_MERGED_RELEASE_WRITER; @@ -37,10 +39,10 @@ public class MergedAndDeprecatedAccessionReleaseFromDBStepConfiguration { ItemStreamWriter accessionWriter; @Bean(MERGED_AND_DEPRECATED_ACCESSIONS_RELEASE_FROM_DB_STEP) - public Step mergedAndDeprecatedAccessionsReleaseFromDBStep(StepBuilderFactory stepBuilderFactory, + public Step mergedAndDeprecatedAccessionsReleaseFromDBStep(JobRepository jobRepository, PlatformTransactionManager transactionManager, SimpleCompletionPolicy chunkSizeCompletionPolicy) { - TaskletStep step = stepBuilderFactory.get(MERGED_AND_DEPRECATED_ACCESSIONS_RELEASE_FROM_DB_STEP) - .chunk(chunkSizeCompletionPolicy) + TaskletStep step = new StepBuilder(MERGED_AND_DEPRECATED_ACCESSIONS_RELEASE_FROM_DB_STEP, jobRepository) + .chunk(chunkSizeCompletionPolicy, transactionManager) .reader(variantReader) .processor(variantProcessor) .writer(accessionWriter) diff --git a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/assembly/AssemblyNameRetrieverTest.java b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/assembly/AssemblyNameRetrieverTest.java index e201c80cb..a53499b9b 100644 --- a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/assembly/AssemblyNameRetrieverTest.java +++ b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/assembly/AssemblyNameRetrieverTest.java @@ -15,37 +15,32 @@ */ package uk.ac.ebi.eva.accession.release.assembly; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; - -import uk.ac.ebi.eva.accession.release.assembly.AssemblyNameRetriever; +import jakarta.xml.bind.JAXBContext; +import jakarta.xml.bind.JAXBException; +import jakarta.xml.bind.Unmarshaller; +import org.junit.jupiter.api.Test; +import uk.ac.ebi.eva.accession.core.utils.PipelineTemporaryFolderUtil; import uk.ac.ebi.eva.accession.release.assembly.AssemblyNameRetriever.EnaAssemblyXml; -import javax.xml.bind.JAXBContext; -import javax.xml.bind.JAXBException; -import javax.xml.bind.Unmarshaller; import java.io.File; import java.io.FileWriter; import java.io.IOException; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertThrows; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; public class AssemblyNameRetrieverTest { - - @Rule - public TemporaryFolder temporaryFolderRule = new TemporaryFolder(); + public PipelineTemporaryFolderUtil temporaryFolderUtil = new PipelineTemporaryFolderUtil(); @Test public void parseXml() throws IOException, JAXBException { - File xml = temporaryFolderRule.newFile(); + File xml = temporaryFolderUtil.newFile(); FileWriter fileWriter = new FileWriter(xml); fileWriter.write("\n" + - "GRCh38.p13\n" + - ""); + "GRCh38.p13\n" + + ""); fileWriter.close(); JAXBContext jaxbContext = JAXBContext.newInstance(EnaAssemblyXml.class); @@ -56,7 +51,7 @@ public void parseXml() throws IOException, JAXBException { @Test public void parseMissingName() throws IOException, JAXBException { - File xml = temporaryFolderRule.newFile(); + File xml = temporaryFolderUtil.newFile(); FileWriter fileWriter = new FileWriter(xml); fileWriter.write(""); fileWriter.close(); @@ -68,24 +63,24 @@ public void parseMissingName() throws IOException, JAXBException { } @Test - public void retrieve() throws IOException, JAXBException { + public void retrieve() { assertEquals("GRCh38.p13", new AssemblyNameRetriever("GCA_000001405.28").getAssemblyName().get()); } @Test - public void retrieveAssemblyWithWrongFormat() throws IOException, JAXBException { + public void retrieveAssemblyWithWrongFormat() { assertThrows(RuntimeException.class, () -> new AssemblyNameRetriever("GCA_wrong_format")); } @Test - public void retrieveNonExistentAssembly() throws IOException, JAXBException { + public void retrieveNonExistentAssembly() { assertFalse(new AssemblyNameRetriever("GCA_000000000.1").getAssemblyName().isPresent()); } @Test public void buildHumanReadableUrl() { assertEquals("https://www.ebi.ac.uk/ena/browser/view/GCA_000001405.28", - new AssemblyNameRetriever("GCA_000001405.28").buildAssemblyHumanReadableUrl()); + new AssemblyNameRetriever("GCA_000001405.28").buildAssemblyHumanReadableUrl()); } @Test diff --git a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/active/AccessionedVariantMongoReaderTest.java b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/active/AccessionedVariantMongoReaderTest.java index fce15442d..14bb1febe 100644 --- a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/active/AccessionedVariantMongoReaderTest.java +++ b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/active/AccessionedVariantMongoReaderTest.java @@ -15,30 +15,28 @@ */ package uk.ac.ebi.eva.accession.release.batch.io.active; -import com.lordofthejars.nosqlunit.annotation.UsingDataSet; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; -import com.mongodb.MongoClient; import com.mongodb.client.AggregateIterable; +import com.mongodb.client.MongoClient; import com.mongodb.client.MongoCollection; import com.mongodb.client.MongoCursor; import com.mongodb.client.MongoDatabase; import org.bson.Document; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.batch.item.ExecutionContext; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.ApplicationContext; +import org.springframework.core.io.ResourceLoader; +import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; - +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.MongoConfiguration; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; +import uk.ac.ebi.eva.accession.core.utils.MongoTestDataLoader; import uk.ac.ebi.eva.accession.release.collectionNames.DbsnpCollectionNames; import uk.ac.ebi.eva.accession.release.test.configuration.MongoTestConfiguration; -import uk.ac.ebi.eva.accession.release.test.rule.FixSpringMongoDbRule; import uk.ac.ebi.eva.commons.core.models.pipeline.Variant; import java.util.ArrayList; @@ -47,10 +45,10 @@ import java.util.Map; import java.util.stream.Collectors; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import static uk.ac.ebi.eva.accession.release.batch.io.VariantMongoAggregationReader.REMAPPED_KEY; import static uk.ac.ebi.eva.accession.release.batch.io.active.AccessionedVariantMongoReader.ALLELES_MATCH_KEY; import static uk.ac.ebi.eva.accession.release.batch.io.active.AccessionedVariantMongoReader.ASSEMBLY_MATCH_KEY; @@ -60,14 +58,10 @@ import static uk.ac.ebi.eva.accession.release.batch.io.active.AccessionedVariantMongoReader.SUPPORTED_BY_EVIDENCE_KEY; import static uk.ac.ebi.eva.accession.release.batch.io.active.AccessionedVariantMongoReader.VARIANT_CLASS_KEY; -@RunWith(SpringRunner.class) +@ExtendWith(SpringExtension.class) @TestPropertySource("classpath:application.properties") -@UsingDataSet(locations = { - "/test-data/dbsnpClusteredVariantEntity.json", - "/test-data/dbsnpSubmittedVariantEntity.json", - "/test-data/submittedVariantEntity.json"}) // includes 1 variant with dbsnp rsid @ContextConfiguration(classes = {MongoConfiguration.class, MongoTestConfiguration.class}) -public class AccessionedVariantMongoReaderTest { +public class AccessionedVariantMongoReaderTest extends MongoTestContainerHelper { private static final String ASSEMBLY_ACCESSION_1 = "GCA_000409795.2"; @@ -126,19 +120,28 @@ public class AccessionedVariantMongoReaderTest { @Autowired private MongoClient mongoClient; - //Required by nosql-unit @Autowired - private ApplicationContext applicationContext; + private MongoTemplate mongoTemplate; - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName(TEST_DB).build()); + @Autowired + private ResourceLoader resourceLoader; - @Before + @BeforeEach public void setUp() throws Exception { + mongoTemplate.getDb().drop(); executionContext = new ExecutionContext(); reader = new AccessionedVariantMongoReader(ASSEMBLY_ACCESSION_1, TAXONOMY_1, mongoClient, TEST_DB, CHUNK_SIZE, - new DbsnpCollectionNames()); + new DbsnpCollectionNames()); + + MongoTestDataLoader mongoTestDataLoader = new MongoTestDataLoader(mongoTemplate, resourceLoader); + mongoTestDataLoader.load("/test-data/dbsnpClusteredVariantEntity.json"); + mongoTestDataLoader.load("/test-data/dbsnpSubmittedVariantEntity.json"); + mongoTestDataLoader.load("/test-data/submittedVariantEntity.json"); + } + + @AfterEach + public void tearDown() { + mongoTemplate.getDb().drop(); } @Test @@ -146,9 +149,7 @@ public void readTestDataMongo() { MongoDatabase db = mongoClient.getDatabase(TEST_DB); MongoCollection collection = db.getCollection(DBSNP_CLUSTERED_VARIANT_ENTITY); - AggregateIterable result = collection.aggregate(reader.buildAggregation()) - .allowDiskUse(true) - .useCursor(true); + AggregateIterable result = collection.aggregate(reader.buildAggregation()).allowDiskUse(true); MongoCursor cursor = result.iterator(); @@ -158,15 +159,15 @@ public void readTestDataMongo() { variants.addAll(reader.getVariants(clusteredVariant)); } assertEquals(EXPECTED_LINES, variants.size()); - } + } @Test - public void reader() throws Exception { + public void reader() { List variants = readIntoList(); assertEquals(EXPECTED_LINES, variants.size()); } - private List readIntoList() throws Exception { + private List readIntoList() { reader.open(executionContext); List allVariants = new ArrayList<>(); List variants; @@ -178,7 +179,7 @@ private List readIntoList() throws Exception { } @Test - public void linkedSubmittedVariants() throws Exception { + public void linkedSubmittedVariants() { Map variants = readIntoMap(); assertEquals(EXPECTED_LINES, variants.size()); assertEquals(3, variants.values().stream().filter(v -> v.getMainId().equals(RS_1)).count()); @@ -188,7 +189,7 @@ public void linkedSubmittedVariants() throws Exception { assertEquals(1, variants.get(RS_2_T_G).getSourceEntries().size()); } - private Map readIntoMap() throws Exception { + private Map readIntoMap() { reader.open(executionContext); Map allVariants = new HashMap<>(); List variants; @@ -207,9 +208,9 @@ private String getStringId(Variant variant) { } @Test - public void queryOtherAssembly() throws Exception { + public void queryOtherAssembly() { reader = new AccessionedVariantMongoReader(ASSEMBLY_ACCESSION_2, TAXONOMY_2, mongoClient, TEST_DB, CHUNK_SIZE, - new DbsnpCollectionNames()); + new DbsnpCollectionNames()); Map variants = readIntoMap(); assertEquals(3, variants.size()); @@ -220,60 +221,60 @@ public void queryOtherAssembly() throws Exception { } @Test - public void snpVariantClassAttribute() throws Exception { + public void snpVariantClassAttribute() { Map variants = readIntoMap(); assertEquals(EXPECTED_LINES, variants.size()); String snpSequenceOntology = "SO:0001483"; assertTrue(variants - .get(RS_1_G_A) - .getSourceEntries() - .stream() - .allMatch(se -> snpSequenceOntology.equals(se.getAttribute(VARIANT_CLASS_KEY)))); + .get(RS_1_G_A) + .getSourceEntries() + .stream() + .allMatch(se -> snpSequenceOntology.equals(se.getAttribute(VARIANT_CLASS_KEY)))); assertTrue(variants - .get(RS_1_G_T) - .getSourceEntries() - .stream() - .allMatch(se -> snpSequenceOntology.equals(se.getAttribute(VARIANT_CLASS_KEY)))); + .get(RS_1_G_T) + .getSourceEntries() + .stream() + .allMatch(se -> snpSequenceOntology.equals(se.getAttribute(VARIANT_CLASS_KEY)))); assertTrue(variants - .get(RS_2_T_G) - .getSourceEntries() - .stream() - .allMatch(se -> snpSequenceOntology.equals(se.getAttribute(VARIANT_CLASS_KEY)))); + .get(RS_2_T_G) + .getSourceEntries() + .stream() + .allMatch(se -> snpSequenceOntology.equals(se.getAttribute(VARIANT_CLASS_KEY)))); } @Test - public void insertionVariantClassAttribute() throws Exception { + public void insertionVariantClassAttribute() { reader = new AccessionedVariantMongoReader(ASSEMBLY_ACCESSION_4, TAXONOMY_4, mongoClient, TEST_DB, CHUNK_SIZE, - new DbsnpCollectionNames()); + new DbsnpCollectionNames()); List variants = readIntoList(); assertEquals(1, variants.size()); String insertionSequenceOntology = "SO:0000667"; assertTrue(variants.get(0) - .getSourceEntries() - .stream() - .allMatch(se -> insertionSequenceOntology.equals(se.getAttribute(VARIANT_CLASS_KEY)))); + .getSourceEntries() + .stream() + .allMatch(se -> insertionSequenceOntology.equals(se.getAttribute(VARIANT_CLASS_KEY)))); } @Test - public void otherVariantClasses() throws Exception { + public void otherVariantClasses() { reader = new AccessionedVariantMongoReader(ASSEMBLY_ACCESSION_5, TAXONOMY_5, mongoClient, TEST_DB, CHUNK_SIZE, - new DbsnpCollectionNames()); + new DbsnpCollectionNames()); List variants = readIntoList(); assertEquals(4, variants.size()); String indelSequenceOntology = "SO:1000032"; String tandemRepeatSequenceOntology = "SO:0000705"; assertEquals(3, variants.stream() - .flatMap(v -> v.getSourceEntries().stream()) - .filter(se -> tandemRepeatSequenceOntology.equals(se.getAttribute(VARIANT_CLASS_KEY))) - .count()); + .flatMap(v -> v.getSourceEntries().stream()) + .filter(se -> tandemRepeatSequenceOntology.equals(se.getAttribute(VARIANT_CLASS_KEY))) + .count()); assertEquals(1, variants.stream() - .flatMap(v -> v.getSourceEntries().stream()) - .filter(se -> indelSequenceOntology.equals(se.getAttribute(VARIANT_CLASS_KEY))) - .count()); + .flatMap(v -> v.getSourceEntries().stream()) + .filter(se -> indelSequenceOntology.equals(se.getAttribute(VARIANT_CLASS_KEY))) + .count()); } @Test - public void studyIdAttribute() throws Exception { + public void studyIdAttribute() { Map variants = readIntoMap(); assertEquals(EXPECTED_LINES, variants.size()); @@ -287,9 +288,9 @@ public void studyIdAttribute() throws Exception { } @Test - public void clusteredVariantWithoutSubmittedVariants() throws Exception { + public void clusteredVariantWithoutSubmittedVariants() { reader = new AccessionedVariantMongoReader(ASSEMBLY_ACCESSION_3, TAXONOMY_3, mongoClient, TEST_DB, CHUNK_SIZE, - new DbsnpCollectionNames()); + new DbsnpCollectionNames()); List variants = readIntoList(); assertEquals(0, variants.size()); } @@ -301,105 +302,105 @@ public void includeValidatedFlag() throws Exception { } @Test - public void includeRemappedFlag() throws Exception { + public void includeRemappedFlag() { List variants = readIntoList(); assertNotEquals(0, variants.size()); List rsToLookFor = variants.stream().filter(v -> v.getMainId().equals("rs8181")) - .collect(Collectors.toList()); + .collect(Collectors.toList()); assertNotEquals(0, rsToLookFor.size()); // Both the SS clustered by the RS 8181 has the "remappedFrom" attribute assertTrue(rsToLookFor.stream().flatMap(v -> v.getSourceEntries().stream()) - .map(se -> se.getAttribute(REMAPPED_KEY)) - .map(Boolean::new) - .allMatch(v -> v.equals(true))); + .map(se -> se.getAttribute(REMAPPED_KEY)) + .map(Boolean::parseBoolean) + .allMatch(v -> v.equals(true))); rsToLookFor = variants.stream().filter(v -> v.getMainId().equals("rs869808637")) - .collect(Collectors.toList()); + .collect(Collectors.toList()); assertNotEquals(0, rsToLookFor.size()); // Only one of the SS clustered by the RS 869808637 has the "remappedFrom" attribute assertTrue(rsToLookFor.stream().flatMap(v -> v.getSourceEntries().stream()) - .map(se -> se.getAttribute(REMAPPED_KEY)) - .map(Boolean::new) - .allMatch(v -> v.equals(false))); + .map(se -> se.getAttribute(REMAPPED_KEY)) + .map(Boolean::parseBoolean) + .allMatch(v -> v.equals(false))); } - private void assertFlagEqualsInAllVariants(String key, boolean value) throws Exception { + private void assertFlagEqualsInAllVariants(String key, boolean value) { List variants = readIntoList(); assertNotEquals(0, variants.size()); assertTrue(variants.stream() - .flatMap(v -> v.getSourceEntries().stream()) - .map(se -> se.getAttribute(key)) - .map(Boolean::new) - .allMatch(v -> v.equals(value))); + .flatMap(v -> v.getSourceEntries().stream()) + .map(se -> se.getAttribute(key)) + .map(Boolean::parseBoolean) + .allMatch(v -> v.equals(value))); } @Test - public void includeAssemblyMatchFlag() throws Exception { + public void includeAssemblyMatchFlag() { assertFlagEqualsInAllVariants(ASSEMBLY_MATCH_KEY, true); } @Test - public void includeAllelesMatchFlag() throws Exception { + public void includeAllelesMatchFlag() { assertFlagEqualsInAllVariants(ALLELES_MATCH_KEY, true); } @Test - public void includeEvidenceFlag() throws Exception { + public void includeEvidenceFlag() { assertFlagEqualsInAllVariants(SUPPORTED_BY_EVIDENCE_KEY, true); } @Test - public void includeValidatedNonDefaultFlag() throws Exception { + public void includeValidatedNonDefaultFlag() { reader = new AccessionedVariantMongoReader(ASSEMBLY_ACCESSION_5, TAXONOMY_5, mongoClient, TEST_DB, CHUNK_SIZE, - new DbsnpCollectionNames()); + new DbsnpCollectionNames()); assertFlagEqualsInAllVariants(SUBMITTED_VARIANT_VALIDATED_KEY, true); assertFlagEqualsInRS(CLUSTERED_VARIANT_VALIDATED_KEY, false, RS_4); assertFlagEqualsInRS(CLUSTERED_VARIANT_VALIDATED_KEY, true, RS_5); } - private void assertFlagEqualsInRS(String key, boolean value, String clusteredVariantAccession) throws Exception { + private void assertFlagEqualsInRS(String key, boolean value, String clusteredVariantAccession) { List variants = readIntoList(); assertNotEquals(0, variants.size()); assertTrue(variants.stream() - .filter(v -> v.getMainId().equals(clusteredVariantAccession)) - .flatMap(v -> v.getSourceEntries().stream()) - .map(se -> se.getAttribute(key)) - .map(Boolean::new) - .allMatch(v -> v.equals(value))); + .filter(v -> v.getMainId().equals(clusteredVariantAccession)) + .flatMap(v -> v.getSourceEntries().stream()) + .map(se -> se.getAttribute(key)) + .map(Boolean::parseBoolean) + .allMatch(v -> v.equals(value))); } @Test - public void includeAssemblyMatchNonDefaultFlag() throws Exception { + public void includeAssemblyMatchNonDefaultFlag() { reader = new AccessionedVariantMongoReader(ASSEMBLY_ACCESSION_4, TAXONOMY_4, mongoClient, TEST_DB, CHUNK_SIZE, - new DbsnpCollectionNames()); + new DbsnpCollectionNames()); assertFlagEqualsInAllVariants(ASSEMBLY_MATCH_KEY, false); } @Test - public void includeAllelesMatchNonDefaultFlag() throws Exception { + public void includeAllelesMatchNonDefaultFlag() { reader = new AccessionedVariantMongoReader(ASSEMBLY_ACCESSION_4, TAXONOMY_4, mongoClient, TEST_DB, CHUNK_SIZE, - new DbsnpCollectionNames()); + new DbsnpCollectionNames()); assertFlagEqualsInAllVariants(ALLELES_MATCH_KEY, false); } @Test - public void includeEvidenceNonDefaultFlag() throws Exception { + public void includeEvidenceNonDefaultFlag() { reader = new AccessionedVariantMongoReader(ASSEMBLY_ACCESSION_5, TAXONOMY_5, mongoClient, TEST_DB, CHUNK_SIZE, - new DbsnpCollectionNames()); + new DbsnpCollectionNames()); assertFlagEqualsInAllVariants(SUPPORTED_BY_EVIDENCE_KEY, false); } /** * Two clustered variants with the same accession but mapped against different locations. Each clustered variant * should only appear with the alleles of the its corresponding submitted variants. - * + *

* This means variants will only be returned by the reader when the clustered and submitted variant have the same location * (contig and start) */ @Test - public void includeOnlyVariantsWithTheSameChromosomeAndStartInRsAndSs() throws Exception { + public void includeOnlyVariantsWithTheSameChromosomeAndStartInRsAndSs() { reader = new AccessionedVariantMongoReader("GCA_000002775.1", 3694, mongoClient, - TEST_DB, CHUNK_SIZE, new DbsnpCollectionNames()); + TEST_DB, CHUNK_SIZE, new DbsnpCollectionNames()); List allVariants = readIntoList(); assertEquals(3, allVariants.size()); @@ -423,31 +424,31 @@ private boolean isVariantPresent(List variants, String chromosome, long * For ambiguous INDELS the start position in the clustered variant and its submitted variants can be different * because the renormalization process is performed only for submitted variants. this will be handled by trying to * match with the exact position or either the one before or after. - * + *

* Variants are represented in different ways by dbSNP and the EVA * dbSNP (start: 7356605, reference: , alternate: GAGCTATGATCTTCGGAAGGAGAAGGAGAAGGAAAAGATTCATGACGTCCACA) * EVA (start: 7356604, reference: , alternate: AGAGCTATGATCTTCGGAAGGAGAAGGAGAAGGAAAAGATTCATGACGTCCAC) - * + *

* FASTA (NC_024803.1:7356603-7356606) TATC - * + *

* dbSNP remove the context nucleotide before an INDEL while the EVA removes the rightmost bases - * + *

* dbSNP: TA(GAGCTATGATCTTCGGAAGGAGAAGGAGAAGGAAAAGATTCATGACGTCCACA)TC, start: 7356605 * EVA: T(AGAGCTATGATCTTCGGAAGGAGAAGGAGAAGGAAAAGATTCATGACGTCCAC)ATC, start: 7356604 - * + *

* For the rs268233057 (start: 7356605) and its ss490570267 (start: 7356604) the submitted variant start will be * used along with its alleles even when the start position does not exactly match. */ @Test - public void includeAmbiguousVariantsWithDifferentStartInSsAndRs() throws Exception { + public void includeAmbiguousVariantsWithDifferentStartInSsAndRs() { reader = new AccessionedVariantMongoReader(ASSEMBLY_ACCESSION_4, TAXONOMY_4, mongoClient, TEST_DB, CHUNK_SIZE, - new DbsnpCollectionNames()); + new DbsnpCollectionNames()); List allVariants = readIntoList(); assertEquals(1, allVariants.size()); assertTrue(isVariantPresent(allVariants, "CM001642.1", 7356604L, "", - "AGAGCTATGATCTTCGGAAGGAGAAGGAGAAGGAAAAGATTCATGACGTCCAC")); + "AGAGCTATGATCTTCGGAAGGAGAAGGAGAAGGAAAAGATTCATGACGTCCAC")); } /* @@ -455,9 +456,9 @@ public void includeAmbiguousVariantsWithDifferentStartInSsAndRs() throws Excepti ensure that the variant list only uses the ss1 entry in ASM1 */ @Test - public void ensureOnlySSInReleaseAssemblyIsUsed() throws Exception { + public void ensureOnlySSInReleaseAssemblyIsUsed() { Variant variantInTwoAssemblies = readIntoList().stream().filter(e -> e.getIds().contains("rs100")) - .findFirst().get(); + .findFirst().get(); // Ensure that only one SS entry is available in the variant that was read assertEquals(1, variantInTwoAssemblies.getSourceEntries().size()); } diff --git a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/active/VariantContextWriterTest.java b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/active/VariantContextWriterTest.java index ac7c66585..49b863d2f 100644 --- a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/active/VariantContextWriterTest.java +++ b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/active/VariantContextWriterTest.java @@ -16,16 +16,15 @@ package uk.ac.ebi.eva.accession.release.batch.io.active; import htsjdk.variant.variantcontext.VariantContext; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; - +import org.junit.jupiter.api.Test; +import org.springframework.batch.item.Chunk; import uk.ac.ebi.eva.accession.core.contig.ContigMapping; import uk.ac.ebi.eva.accession.core.contig.ContigNaming; import uk.ac.ebi.eva.accession.core.contig.ContigSynonyms; +import uk.ac.ebi.eva.accession.core.utils.PipelineTemporaryFolderUtil; import uk.ac.ebi.eva.accession.release.batch.io.contig.ContigWriter; -import uk.ac.ebi.eva.accession.release.parameters.ReportPathResolver; import uk.ac.ebi.eva.accession.release.batch.processors.VariantToVariantContextProcessor; +import uk.ac.ebi.eva.accession.release.parameters.ReportPathResolver; import uk.ac.ebi.eva.commons.core.models.pipeline.Variant; import uk.ac.ebi.eva.commons.core.models.pipeline.VariantSourceEntry; @@ -45,8 +44,9 @@ import java.util.stream.Collectors; import java.util.stream.Stream; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import static uk.ac.ebi.eva.accession.release.batch.io.active.AccessionedVariantMongoReader.ALLELES_MATCH_KEY; import static uk.ac.ebi.eva.accession.release.batch.io.active.AccessionedVariantMongoReader.ASSEMBLY_MATCH_KEY; import static uk.ac.ebi.eva.accession.release.batch.io.active.AccessionedVariantMongoReader.CLUSTERED_VARIANT_VALIDATED_KEY; @@ -98,22 +98,21 @@ public class VariantContextWriterTest { // inner string without space nor angle brackets, surrounded by angle brackets private static final String SYMBOLIC_ALLELE_REGEX = "<[^<> ]+>"; - @Rule - public TemporaryFolder temporaryFolder = new TemporaryFolder(); + public PipelineTemporaryFolderUtil temporaryFolderUtil = new PipelineTemporaryFolderUtil(); private static final String DATA_LINES_REGEX = "^[^#].*"; @Test public void basicWrite() throws Exception { - File outputFolder = temporaryFolder.newFolder(); + File outputFolder = temporaryFolderUtil.newFolder(); File output = assertWriteVcf(outputFolder, - buildVariant(GENBANK_ACCESSION_1, 1000, "C", "A", SNP_SEQUENCE_ONTOLOGY, STUDY_1)); + buildVariant(GENBANK_ACCESSION_1, 1000, "C", "A", SNP_SEQUENCE_ONTOLOGY, STUDY_1)); } - private Variant buildVariant(String chr, int start, String reference, String alternate,String sequenceOntologyTerm, + private Variant buildVariant(String chr, int start, String reference, String alternate, String sequenceOntologyTerm, String... studies) { return buildVariant(chr, start, reference, alternate, sequenceOntologyTerm, false, false, true, true, true, - studies); + studies); } private Variant buildVariant(String chr, int start, String reference, String alternate, String sequenceOntologyTerm, @@ -138,7 +137,7 @@ private Variant buildVariant(String chr, int start, String reference, String alt private File assertWriteVcf(File outputFolder, Variant... variants) throws Exception { Path reportPath = ReportPathResolver.getDbsnpCurrentIdsReportPath(outputFolder.getAbsolutePath(), REFERENCE_ASSEMBLY); String activeContigsFilePath = ContigWriter.getDbsnpActiveContigsFilePath(reportPath.toFile().getParent(), - REFERENCE_ASSEMBLY); + REFERENCE_ASSEMBLY); VariantContextWriter writer = new VariantContextWriter(reportPath, REFERENCE_ASSEMBLY, activeContigsFilePath); writer.open(null); @@ -150,7 +149,7 @@ private File assertWriteVcf(File outputFolder, Variant... variants) throws Excep List variantContexts = Stream.of(variants).map(variantToVariantContextProcessor::process).collect(Collectors.toList()); - writer.write(variantContexts); + writer.write(new Chunk<>(variantContexts)); writer.close(); @@ -162,14 +161,14 @@ private File assertWriteVcf(File outputFolder, Variant... variants) throws Excep @Test public void checkReference() throws Exception { - File outputFolder = temporaryFolder.newFolder(); + File outputFolder = temporaryFolderUtil.newFolder(); File output = assertWriteVcf(outputFolder, - buildVariant(GENBANK_ACCESSION_1, 1000, "C", "A", SNP_SEQUENCE_ONTOLOGY, STUDY_1)); + buildVariant(GENBANK_ACCESSION_1, 1000, "C", "A", SNP_SEQUENCE_ONTOLOGY, STUDY_1)); List referenceLines = grepFile(output, "^##reference.*"); assertEquals(1, referenceLines.size()); assertEquals("##reference=", referenceLines.get(0)); + + REFERENCE_ASSEMBLY + ">", referenceLines.get(0)); } private List grepFile(File file, String regex) throws IOException { @@ -187,7 +186,7 @@ private List grepFile(File file, String regex) throws IOException { @Test public void checkMetadataSection() throws Exception { - File outputFolder = temporaryFolder.newFolder(); + File outputFolder = temporaryFolderUtil.newFolder(); FileWriter fileWriter = new FileWriter( ContigWriter.getDbsnpActiveContigsFilePath(outputFolder.getAbsolutePath(), REFERENCE_ASSEMBLY)); String contig = "CM0001.1,Chr1"; @@ -195,7 +194,7 @@ public void checkMetadataSection() throws Exception { fileWriter.close(); File output = assertWriteVcf(outputFolder, - buildVariant(GENBANK_ACCESSION_1, 1000, "C", "A", SNP_SEQUENCE_ONTOLOGY, STUDY_1)); + buildVariant(GENBANK_ACCESSION_1, 1000, "C", "A", SNP_SEQUENCE_ONTOLOGY, STUDY_1)); List metadataLines = grepFile(output, "^##.*"); assertEquals(11, metadataLines.size()); @@ -207,9 +206,9 @@ public void checkMetadataSection() throws Exception { @Test public void checkAccession() throws Exception { - File outputFolder = temporaryFolder.newFolder(); + File outputFolder = temporaryFolderUtil.newFolder(); File output = assertWriteVcf(outputFolder, - buildVariant(GENBANK_ACCESSION_1, 1000, "C", "A", SNP_SEQUENCE_ONTOLOGY, STUDY_1)); + buildVariant(GENBANK_ACCESSION_1, 1000, "C", "A", SNP_SEQUENCE_ONTOLOGY, STUDY_1)); List dataLines = grepFileContains(output, ID); assertEquals(1, dataLines.size()); @@ -217,9 +216,9 @@ public void checkAccession() throws Exception { @Test public void checkColumns() throws Exception { - File outputFolder = temporaryFolder.newFolder(); + File outputFolder = temporaryFolderUtil.newFolder(); File output = assertWriteVcf(outputFolder, - buildVariant(GENBANK_ACCESSION_1, 1000, "C", "A", SNP_SEQUENCE_ONTOLOGY, STUDY_1)); + buildVariant(GENBANK_ACCESSION_1, 1000, "C", "A", SNP_SEQUENCE_ONTOLOGY, STUDY_1)); List dataLines = grepFileContains(output, ID); assertEquals(1, dataLines.size()); @@ -228,10 +227,10 @@ public void checkColumns() throws Exception { @Test public void checkStudies() throws Exception { - File outputFolder = temporaryFolder.newFolder(); + File outputFolder = temporaryFolderUtil.newFolder(); File output = assertWriteVcf(outputFolder, - buildVariant(GENBANK_ACCESSION_1, 1000, "C", "A", SNP_SEQUENCE_ONTOLOGY, STUDY_1, - STUDY_2)); + buildVariant(GENBANK_ACCESSION_1, 1000, "C", "A", SNP_SEQUENCE_ONTOLOGY, STUDY_1, + STUDY_2)); List metadataLines = grepFileContains(output, STUDY_1); assertEquals(1, metadataLines.size()); @@ -252,10 +251,10 @@ public void checkSnpSequenceOntology() throws Exception { } private void checkSequenceOntology(String sequenceOntology, String reference, String alternate) throws Exception { - File outputFolder = temporaryFolder.newFolder(); + File outputFolder = temporaryFolderUtil.newFolder(); File output = assertWriteVcf(outputFolder, - buildVariant(GENBANK_ACCESSION_1, 1000, reference, alternate, sequenceOntology, - STUDY_1, STUDY_2)); + buildVariant(GENBANK_ACCESSION_1, 1000, reference, alternate, sequenceOntology, + STUDY_1, STUDY_2)); String dataWithVariantClassRegex = createRegexSurroundedByTabOrSemicolonInDataLine( VARIANT_CLASS_KEY + "=SO:[0-9]+"); @@ -292,9 +291,9 @@ public void checkTandemRepeatSequenceOntology() throws Exception { checkSequenceOntology(TANDEM_REPEAT_SEQUENCE_ONTOLOGY, "C", "CAGAG"); } - @Test(expected = IllegalArgumentException.class) - public void checkSequenceAlterationSequenceOntology() throws Exception { - checkSequenceOntology(SEQUENCE_ALTERATION_SEQUENCE_ONTOLOGY, "(ADL260)", "(LEI0062)"); + @Test + public void checkSequenceAlterationSequenceOntology() { + assertThrows(IllegalArgumentException.class, () -> checkSequenceOntology(SEQUENCE_ALTERATION_SEQUENCE_ONTOLOGY, "(ADL260)", "(LEI0062)")); } @Test @@ -304,7 +303,7 @@ public void checkMnvSequenceOntology() throws Exception { @Test public void checkSeveralVariants() throws Exception { - File outputFolder = temporaryFolder.newFolder(); + File outputFolder = temporaryFolderUtil.newFolder(); int position1 = 1003; int position2 = 1003; int position3 = 1002; @@ -313,12 +312,12 @@ public void checkSeveralVariants() throws Exception { String alternate3 = "G"; File output = assertWriteVcf(outputFolder, - buildVariant(GENBANK_ACCESSION_1, position1, "C", alternate1, - SNP_SEQUENCE_ONTOLOGY, STUDY_1), - buildVariant(GENBANK_ACCESSION_1, position2, "C", alternate2, - SNP_SEQUENCE_ONTOLOGY, STUDY_1), - buildVariant(GENBANK_ACCESSION_1, position3, "C", alternate3, - SNP_SEQUENCE_ONTOLOGY, STUDY_1)); + buildVariant(GENBANK_ACCESSION_1, position1, "C", alternate1, + SNP_SEQUENCE_ONTOLOGY, STUDY_1), + buildVariant(GENBANK_ACCESSION_1, position2, "C", alternate2, + SNP_SEQUENCE_ONTOLOGY, STUDY_1), + buildVariant(GENBANK_ACCESSION_1, position3, "C", alternate3, + SNP_SEQUENCE_ONTOLOGY, STUDY_1)); List dataLines = grepFileContains(output, ID); assertEquals(3, dataLines.size()); @@ -334,29 +333,29 @@ public void checkSeveralVariants() throws Exception { @Test public void checkStandardNucleotides() throws Exception { - File outputFolder = temporaryFolder.newFolder(); + File outputFolder = temporaryFolderUtil.newFolder(); File output = assertWriteVcf(outputFolder, - buildVariant(GENBANK_ACCESSION_1, 1000, "C", "NACTG", SNP_SEQUENCE_ONTOLOGY, - STUDY_1)); + buildVariant(GENBANK_ACCESSION_1, 1000, "C", "NACTG", SNP_SEQUENCE_ONTOLOGY, + STUDY_1)); List dataLines = grepFileContains(output, ID); assertEquals(1, dataLines.size()); } - @Test(expected = IllegalArgumentException.class) + @Test public void throwIfNonStandardNucleotides() throws Exception { - File outputFolder = temporaryFolder.newFolder(); - File output = assertWriteVcf(outputFolder, - buildVariant(GENBANK_ACCESSION_1, 1000, "C", "U", SNP_SEQUENCE_ONTOLOGY, STUDY_1)); + File outputFolder = temporaryFolderUtil.newFolder(); + assertThrows(IllegalArgumentException.class, () -> assertWriteVcf(outputFolder, + buildVariant(GENBANK_ACCESSION_1, 1000, "C", "U", SNP_SEQUENCE_ONTOLOGY, STUDY_1))); } @Test public void checkFlagsAreNotPresentWhenDefaultValues() throws Exception { - File outputFolder = temporaryFolder.newFolder(); + File outputFolder = temporaryFolderUtil.newFolder(); File output = assertWriteVcf(outputFolder, - buildVariant(GENBANK_ACCESSION_1, 1000, "C", "G", SNP_SEQUENCE_ONTOLOGY, false, - false, true, true, true, - STUDY_1)); + buildVariant(GENBANK_ACCESSION_1, 1000, "C", "G", SNP_SEQUENCE_ONTOLOGY, false, + false, true, true, true, + STUDY_1)); String dataLinesWithFlagsRegex = DATA_LINES_REGEX + "(" + CLUSTERED_VARIANT_VALIDATED_KEY + "|" + ALLELES_MATCH_KEY @@ -373,11 +372,11 @@ public void checkNonDefaultValidatedFlag() throws Exception { } private void assertFlagIsPresent(String flagRegex) throws Exception { - File outputFolder = temporaryFolder.newFolder(); + File outputFolder = temporaryFolderUtil.newFolder(); File output = assertWriteVcf(outputFolder, - buildVariant(GENBANK_ACCESSION_1, 1000, "C", "G", SNP_SEQUENCE_ONTOLOGY, true, - true, false, false, false, - STUDY_1)); + buildVariant(GENBANK_ACCESSION_1, 1000, "C", "G", SNP_SEQUENCE_ONTOLOGY, true, + true, false, false, false, + STUDY_1)); String dataLinesWithValidatedRegex = createRegexSurroundedByTabOrSemicolonInDataLine(flagRegex); List dataLines; @@ -429,7 +428,7 @@ private void assertSeveralFlagValues(String flagKey, boolean firstValue, boolean VariantSourceEntry sourceEntry2 = new VariantSourceEntry(STUDY_2, FILE_ID); sourceEntry2.addAttribute(flagKey, Boolean.toString(secondValue)); variant.addSourceEntry(sourceEntry2); - File outputFolder = temporaryFolder.newFolder(); + File outputFolder = temporaryFolderUtil.newFolder(); File output = assertWriteVcf(outputFolder, variant); @@ -460,9 +459,9 @@ public void writeNamedInsertion() throws Exception { } private void assertNamedVariant(String reference, String alternate) throws Exception { - File outputFolder = temporaryFolder.newFolder(); + File outputFolder = temporaryFolderUtil.newFolder(); File output = assertWriteVcf(outputFolder, buildVariant(GENBANK_ACCESSION_1, 1000, reference, alternate, - SEQUENCE_ALTERATION_SEQUENCE_ONTOLOGY, STUDY_1)); + SEQUENCE_ALTERATION_SEQUENCE_ONTOLOGY, STUDY_1)); List dataLines = grepFile(output, DATA_LINES_REGEX); assertEquals(1, dataLines.size()); @@ -476,35 +475,35 @@ public void writeNamedDeletion() throws Exception { assertNamedVariant("A", "<1190_BP_DEL>"); } - @Test(expected = IllegalArgumentException.class) + @Test public void throwIfNamedReference() throws Exception { - File outputFolder = temporaryFolder.newFolder(); - File output = assertWriteVcf(outputFolder, buildVariant(GENBANK_ACCESSION_1, 1000, "<1190_BP_DEL>", "A", - SEQUENCE_ALTERATION_SEQUENCE_ONTOLOGY, STUDY_1)); + File outputFolder = temporaryFolderUtil.newFolder(); + assertThrows(IllegalArgumentException.class, () -> assertWriteVcf(outputFolder, buildVariant(GENBANK_ACCESSION_1, 1000, "<1190_BP_DEL>", "A", + SEQUENCE_ALTERATION_SEQUENCE_ONTOLOGY, STUDY_1))); } @Test public void writeStudyWithInvalidCharacters() throws Exception { - File outputFolder = temporaryFolder.newFolder(); + File outputFolder = temporaryFolderUtil.newFolder(); File output = assertWriteVcf(outputFolder, - buildVariant(GENBANK_ACCESSION_1, 1000, "A", "T", - SEQUENCE_ALTERATION_SEQUENCE_ONTOLOGY, - "study_id=weird, yes; but not impossible in dbSNP"), - buildVariant(GENBANK_ACCESSION_1, 2000, "A", "T", - SEQUENCE_ALTERATION_SEQUENCE_ONTOLOGY, - "one =study= ", - ";; extra study ;;")); + buildVariant(GENBANK_ACCESSION_1, 1000, "A", "T", + SEQUENCE_ALTERATION_SEQUENCE_ONTOLOGY, + "study_id=weird, yes; but not impossible in dbSNP"), + buildVariant(GENBANK_ACCESSION_1, 2000, "A", "T", + SEQUENCE_ALTERATION_SEQUENCE_ONTOLOGY, + "one =study= ", + ";; extra study ;;")); List dataLines = grepFile(output, DATA_LINES_REGEX); assertEquals(2, dataLines.size()); String[] columns = dataLines.get(0).split("\t"); assertEquals(Arrays.asList("study_id_weird__yes__but_not_impossible_in_dbSNP"), - parseInfoFields(columns[INFO_COLUMN]).get(STUDY_ID_KEY)); + parseInfoFields(columns[INFO_COLUMN]).get(STUDY_ID_KEY)); columns = dataLines.get(1).split("\t"); assertEquals(new HashSet<>(Arrays.asList("one__study__", "___extra_study___")), - new HashSet<>(parseInfoFields(columns[INFO_COLUMN]).get(STUDY_ID_KEY))); + new HashSet<>(parseInfoFields(columns[INFO_COLUMN]).get(STUDY_ID_KEY))); } private HashMap> parseInfoFields(String column) { diff --git a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/contig/ContigMongoReaderTest.java b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/contig/ContigMongoReaderTest.java index 4df8bf83d..17056b906 100644 --- a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/contig/ContigMongoReaderTest.java +++ b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/contig/ContigMongoReaderTest.java @@ -16,42 +16,36 @@ package uk.ac.ebi.eva.accession.release.batch.io.contig; -import com.lordofthejars.nosqlunit.annotation.UsingDataSet; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; -import com.mongodb.MongoClient; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; +import com.mongodb.client.MongoClient; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.batch.item.ExecutionContext; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.ApplicationContext; +import org.springframework.core.io.ResourceLoader; +import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; - +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.MongoConfiguration; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; +import uk.ac.ebi.eva.accession.core.utils.MongoTestDataLoader; import uk.ac.ebi.eva.accession.release.collectionNames.DbsnpCollectionNames; import uk.ac.ebi.eva.accession.release.collectionNames.EvaCollectionNames; import uk.ac.ebi.eva.accession.release.test.configuration.MongoTestConfiguration; -import uk.ac.ebi.eva.accession.release.test.rule.FixSpringMongoDbRule; import java.util.ArrayList; import java.util.Arrays; import java.util.HashSet; import java.util.List; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; -@RunWith(SpringRunner.class) +@ExtendWith(SpringExtension.class) @TestPropertySource("classpath:application.properties") -@UsingDataSet(locations = { - "/test-data/dbsnpClusteredVariantEntity.json", - "/test-data/dbsnpClusteredVariantOperationEntity.json", - "/test-data/submittedVariantEntity.json" -}) @ContextConfiguration(classes = {MongoConfiguration.class, MongoTestConfiguration.class}) -public class ContigMongoReaderTest { +public class ContigMongoReaderTest extends MongoTestContainerHelper { private static final String TEST_DB = "test-db"; @@ -62,13 +56,25 @@ public class ContigMongoReaderTest { @Autowired private MongoClient mongoClient; - //Required by nosql-unit @Autowired - private ApplicationContext applicationContext; + private MongoTemplate mongoTemplate; + + @Autowired + private ResourceLoader resourceLoader; - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName(TEST_DB).build()); + @BeforeEach + public void setUp() { + mongoTemplate.getDb().drop(); + MongoTestDataLoader mongoTestDataLoader = new MongoTestDataLoader(mongoTemplate, resourceLoader); + mongoTestDataLoader.load("/test-data/dbsnpClusteredVariantEntity.json"); + mongoTestDataLoader.load("/test-data/dbsnpClusteredVariantOperationEntity.json"); + mongoTestDataLoader.load("/test-data/submittedVariantEntity.json"); + } + + @AfterEach + public void tearDown() { + mongoTemplate.getDb().drop(); + } @Test public void basicActiveContigsRead() { @@ -86,7 +92,7 @@ public void basicActiveContigsRead() { @Test public void basicMergedContigsRead() { ContigMongoReader reader = ContigMongoReader.mergedContigReader(ASSEMBLY_ACCESSION, mongoClient, - TEST_DB, new DbsnpCollectionNames()); + TEST_DB, new DbsnpCollectionNames()); reader.open(new ExecutionContext()); String contig; List contigs = new ArrayList<>(); @@ -99,7 +105,7 @@ public void basicMergedContigsRead() { @Test public void basicMultimapContigsRead() { ContigMongoReader reader = ContigMongoReader.multimapContigReader(ASSEMBLY_ACCESSION, mongoClient, TEST_DB, - new DbsnpCollectionNames()); + new DbsnpCollectionNames()); reader.open(new ExecutionContext()); String contig; List contigs = new ArrayList<>(); diff --git a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/contig/ContigWriterTest.java b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/contig/ContigWriterTest.java index 689c520e3..31231b82b 100644 --- a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/contig/ContigWriterTest.java +++ b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/contig/ContigWriterTest.java @@ -15,14 +15,12 @@ */ package uk.ac.ebi.eva.accession.release.batch.io.contig; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; - +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.batch.item.Chunk; import uk.ac.ebi.eva.accession.core.contig.ContigMapping; import uk.ac.ebi.eva.accession.core.contig.ContigSynonyms; -import uk.ac.ebi.eva.accession.release.batch.io.contig.ContigWriter; +import uk.ac.ebi.eva.accession.core.utils.PipelineTemporaryFolderUtil; import java.io.BufferedReader; import java.io.File; @@ -32,7 +30,8 @@ import java.util.List; import static junit.framework.TestCase.assertTrue; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; public class ContigWriterTest { @@ -60,28 +59,27 @@ public class ContigWriterTest { private File output; - @Rule - public TemporaryFolder temporaryFolderRule = new TemporaryFolder(); + public PipelineTemporaryFolderUtil temporaryFolderUtil = new PipelineTemporaryFolderUtil(); - @Before + @BeforeEach public void setUp() throws Exception { - output = temporaryFolderRule.newFile(); + output = temporaryFolderUtil.newFile(); } @Test public void write() throws Exception { ContigMapping contigMapping = new ContigMapping(Arrays.asList( new ContigSynonyms(SEQUENCE_NAME_1, "assembled-molecule", "1", GENBANK_ACCESSION_1, REFSEQ_ACCESSION_1, - "ucsc1", true), + "ucsc1", true), new ContigSynonyms(SEQUENCE_NAME_2, "assembled-molecule", "2", GENBANK_ACCESSION_2, REFSEQ_ACCESSION_2, - "ucsc2", false), + "ucsc2", false), new ContigSynonyms(SEQUENCE_NAME_3, "assembled-molecule", "3", GENBANK_ACCESSION_3, "na", "ucsc3", - false))); + false))); ContigWriter contigWriter = new ContigWriter(output, contigMapping); contigWriter.open(null); List contigs = Arrays.asList(GENBANK_ACCESSION_1, GENBANK_ACCESSION_2, GENBANK_ACCESSION_3); - contigWriter.write(contigs); + contigWriter.write(new Chunk<>(contigs)); contigWriter.close(); assertEquals(contigs.size(), numberOfLines(output)); @@ -110,16 +108,16 @@ private void assertContigFileContent(File file, List expectedLines) thro public void useSequenceNameIfContigIsRefSeqAccession() throws IOException { ContigMapping contigMapping = new ContigMapping(Arrays.asList( new ContigSynonyms(SEQUENCE_NAME_1, "assembled-molecule", "1", GENBANK_ACCESSION_1, REFSEQ_ACCESSION_1, - "ucsc1", true), + "ucsc1", true), new ContigSynonyms(SEQUENCE_NAME_2, "assembled-molecule", "2", GENBANK_ACCESSION_2, REFSEQ_ACCESSION_2, - "ucsc2", false), + "ucsc2", false), new ContigSynonyms(SEQUENCE_NAME_3, "assembled-molecule", "3", GENBANK_ACCESSION_3, "na", "ucsc3", - false))); + false))); ContigWriter contigWriter = new ContigWriter(output, contigMapping); contigWriter.open(null); List contigs = Arrays.asList(REFSEQ_ACCESSION_1, REFSEQ_ACCESSION_2, GENBANK_ACCESSION_3); - contigWriter.write(contigs); + contigWriter.write(new Chunk<>(contigs)); contigWriter.close(); assertEquals(contigs.size(), numberOfLines(output)); @@ -131,64 +129,64 @@ public void useSequenceNameIfContigIsRefSeqAccession() throws IOException { /** * This will happen a contig is null */ - @Test(expected = IllegalArgumentException.class) + @Test public void throwExceptionIfNullContig() { ContigMapping contigMapping = new ContigMapping(Arrays.asList( new ContigSynonyms(SEQUENCE_NAME_1, "assembled-molecule", "1", GENBANK_ACCESSION_1, REFSEQ_ACCESSION_1, - "ucsc1", true))); + "ucsc1", true))); ContigWriter contigWriter = new ContigWriter(output, contigMapping); contigWriter.open(null); String nullGenbankAccession = null; - contigWriter.write(Arrays.asList(GENBANK_ACCESSION_1, nullGenbankAccession)); + assertThrows(IllegalArgumentException.class, () -> contigWriter.write(Chunk.of(GENBANK_ACCESSION_1, nullGenbankAccession))); contigWriter.close(); } /** * This will happen a contig is empty */ - @Test(expected = IllegalArgumentException.class) + @Test public void throwExceptionIfEmptyContig() { ContigMapping contigMapping = new ContigMapping(Arrays.asList( new ContigSynonyms(SEQUENCE_NAME_1, "assembled-molecule", "1", GENBANK_ACCESSION_1, REFSEQ_ACCESSION_1, - "ucsc1", true))); + "ucsc1", true))); ContigWriter contigWriter = new ContigWriter(output, contigMapping); contigWriter.open(null); String emptyGenbankAccession = ""; - contigWriter.write(Arrays.asList(GENBANK_ACCESSION_1, emptyGenbankAccession)); + assertThrows(IllegalArgumentException.class, () -> contigWriter.write(Chunk.of(GENBANK_ACCESSION_1, emptyGenbankAccession))); contigWriter.close(); } /** * This will happen if the assembly report does not have a value for sequence name in at least one row */ - @Test(expected = IllegalArgumentException.class) + @Test public void throwExceptionIfEmptySequenceName() { ContigMapping contigMapping = new ContigMapping(Arrays.asList( new ContigSynonyms(SEQUENCE_NAME_1, "assembled-molecule", "1", GENBANK_ACCESSION_1, REFSEQ_ACCESSION_1, - "ucsc1", true), + "ucsc1", true), new ContigSynonyms(EMPTY_STRING, "assembled-molecule", "4", GENBANK_ACCESSION_EMPTY_SEQUENCE_NAME, "na", - "ucsc4", false))); + "ucsc4", false))); ContigWriter contigWriter = new ContigWriter(output, contigMapping); contigWriter.open(null); - contigWriter.write(Arrays.asList(GENBANK_ACCESSION_1, GENBANK_ACCESSION_EMPTY_SEQUENCE_NAME)); + assertThrows(IllegalArgumentException.class, () -> contigWriter.write(Chunk.of(GENBANK_ACCESSION_1, GENBANK_ACCESSION_EMPTY_SEQUENCE_NAME))); contigWriter.close(); } /** * This will happen if there is any contig in mongo that is not in the assembly report */ - @Test(expected = IllegalArgumentException.class) + @Test public void throwExceptionIfContigNotInAssemblyReport() { ContigMapping contigMapping = new ContigMapping(Arrays.asList( new ContigSynonyms(SEQUENCE_NAME_1, "assembled-molecule", "1", GENBANK_ACCESSION_1, REFSEQ_ACCESSION_1, - "ucsc1", true))); + "ucsc1", true))); ContigWriter contigWriter = new ContigWriter(output, contigMapping); contigWriter.open(null); - contigWriter.write(Arrays.asList(GENBANK_ACCESSION_1, GENBANK_ACCESSION_NOT_IN_ASSEMBLY_REPORT)); + assertThrows(IllegalArgumentException.class, () -> contigWriter.write(Chunk.of(GENBANK_ACCESSION_1, GENBANK_ACCESSION_NOT_IN_ASSEMBLY_REPORT))); contigWriter.close(); } } \ No newline at end of file diff --git a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/deprecated/DeprecatedVariantAccessionWriterTest.java b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/deprecated/DeprecatedVariantAccessionWriterTest.java index 690353127..f22a68e98 100644 --- a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/deprecated/DeprecatedVariantAccessionWriterTest.java +++ b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/deprecated/DeprecatedVariantAccessionWriterTest.java @@ -15,31 +15,28 @@ */ package uk.ac.ebi.eva.accession.release.batch.io.deprecated; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; - +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.batch.item.Chunk; +import uk.ac.ebi.eva.accession.core.utils.PipelineTemporaryFolderUtil; import uk.ac.ebi.eva.commons.core.models.pipeline.Variant; import java.io.BufferedReader; import java.io.File; import java.io.FileReader; import java.io.IOException; -import java.util.Arrays; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; public class DeprecatedVariantAccessionWriterTest { private DeprecatedVariantAccessionWriter deprecatedVariantAccessionWriter; - @Rule - public TemporaryFolder temporaryFolderRule = new TemporaryFolder(); + public PipelineTemporaryFolderUtil temporaryFolderUtil = new PipelineTemporaryFolderUtil(); - @Before + @BeforeEach public void setUp() throws Exception { - File output = temporaryFolderRule.newFile(); + File output = temporaryFolderUtil.newFile(); deprecatedVariantAccessionWriter = new DeprecatedVariantAccessionWriter(output.toPath()); } @@ -53,7 +50,7 @@ public void write() throws Exception { variant3.setMainId("rs3"); deprecatedVariantAccessionWriter.open(null); - deprecatedVariantAccessionWriter.write(Arrays.asList(variant1, variant2, variant3)); + deprecatedVariantAccessionWriter.write(Chunk.of(variant1, variant2, variant3)); deprecatedVariantAccessionWriter.close(); assertEquals(3, numberOfLines(deprecatedVariantAccessionWriter.getOutput())); diff --git a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/deprecated/DeprecatedVariantMongoReaderTest.java b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/deprecated/DeprecatedVariantMongoReaderTest.java index b4f902350..75adc3d34 100644 --- a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/deprecated/DeprecatedVariantMongoReaderTest.java +++ b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/deprecated/DeprecatedVariantMongoReaderTest.java @@ -16,21 +16,17 @@ package uk.ac.ebi.eva.accession.release.batch.io.deprecated; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; -import com.mongodb.MongoClient; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; +import com.mongodb.client.MongoClient; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.batch.item.ExecutionContext; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.ApplicationContext; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.ampt2d.commons.accession.core.models.EventType; import uk.ac.ebi.ampt2d.commons.accession.hashing.SHA1HashingFunction; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.MongoConfiguration; @@ -44,9 +40,9 @@ import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpSubmittedVariantOperationEntity; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; import uk.ac.ebi.eva.accession.core.summary.ClusteredVariantSummaryFunction; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.release.collectionNames.DbsnpCollectionNames; import uk.ac.ebi.eva.accession.release.test.configuration.MongoTestConfiguration; -import uk.ac.ebi.eva.accession.release.test.rule.FixSpringMongoDbRule; import uk.ac.ebi.eva.commons.core.models.pipeline.Variant; import java.util.ArrayList; @@ -57,13 +53,13 @@ import java.util.stream.Collectors; import java.util.stream.Stream; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; -@RunWith(SpringRunner.class) +@ExtendWith(SpringExtension.class) @TestPropertySource("classpath:application.properties") @ContextConfiguration(classes = {MongoConfiguration.class, MongoTestConfiguration.class}) -public class DeprecatedVariantMongoReaderTest { +public class DeprecatedVariantMongoReaderTest extends MongoTestContainerHelper { private static final String TEST_DB = "test-db"; @@ -83,22 +79,14 @@ public class DeprecatedVariantMongoReaderTest { @Autowired private MongoTemplate mongoTemplate; - //Required by nosql-unit - @Autowired - private ApplicationContext applicationContext; - - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName(TEST_DB).build()); - private DeprecatedVariantMongoReader reader; - @Before + @BeforeEach public void setUp() { this.mongoTemplate.getDb().drop(); } - @After + @AfterEach public void tearDown() { this.mongoTemplate.getDb().drop(); this.reader.close(); @@ -128,22 +116,22 @@ public void testOnlySpecifiedTaxVariantsRead() { // create RS this.mongoTemplate.insert(Stream.of(ss1, ss2, ss3, ss4).map(ss -> { DbsnpSubmittedVariantOperationEntity dbsnpSvoeObj = new DbsnpSubmittedVariantOperationEntity(); - dbsnpSvoeObj.fill( EventType.UPDATED, ss.getAccession(), - "Declustered: None of the variant alleles match the reference allele.", - Arrays.asList(new DbsnpSubmittedVariantInactiveEntity(ss))); + dbsnpSvoeObj.fill(EventType.UPDATED, ss.getAccession(), + "Declustered: None of the variant alleles match the reference allele.", + Arrays.asList(new DbsnpSubmittedVariantInactiveEntity(ss))); return dbsnpSvoeObj; }).collect(Collectors.toList()), DbsnpSubmittedVariantOperationEntity.class); this.mongoTemplate.insert(Stream.of(rs1, rs2, rs3, rs4).map(rs -> { DbsnpClusteredVariantOperationEntity dbsnpCvoeObj = new DbsnpClusteredVariantOperationEntity(); - dbsnpCvoeObj.fill( EventType.DEPRECATED, rs.getAccession(), - "Clustered variant completely declustered", - Arrays.asList(new DbsnpClusteredVariantInactiveEntity(rs))); + dbsnpCvoeObj.fill(EventType.DEPRECATED, rs.getAccession(), + "Clustered variant completely declustered", + Arrays.asList(new DbsnpClusteredVariantInactiveEntity(rs))); return dbsnpCvoeObj; }).collect(Collectors.toList()), DbsnpClusteredVariantOperationEntity.class); // Test records returned when the mongo reader is given TAXONOMY_1 this.reader = new DeprecatedVariantMongoReader(ASSEMBLY, TAXONOMY_1, mongoClient, TEST_DB, CHUNK_SIZE, - new DbsnpCollectionNames()); + new DbsnpCollectionNames()); this.reader.open(new ExecutionContext()); List deprecatedVariants = this.readIntoList(); assertEquals(1, deprecatedVariants.size()); @@ -153,7 +141,7 @@ public void testOnlySpecifiedTaxVariantsRead() { // Note rs3 is returned even though in Mongo it was originally issued in TAXONOMY_1 // This is because there is a SS record in TAXONOMY_2 that was declustered from rs3 this.reader = new DeprecatedVariantMongoReader(ASSEMBLY, TAXONOMY_2, mongoClient, TEST_DB, CHUNK_SIZE, - new DbsnpCollectionNames()); + new DbsnpCollectionNames()); this.reader.open(new ExecutionContext()); deprecatedVariants = this.readIntoList(); assertEquals(2, deprecatedVariants.size()); @@ -171,22 +159,22 @@ public void testOnlySpecifiedTaxVariantsRead() { private DbsnpSubmittedVariantEntity createSS(String assembly, int taxonomy, Long ssAccession, Long rsAccession, Long start, String reference, String alternate) { return new DbsnpSubmittedVariantEntity(ssAccession, "hash" + ssAccession, assembly, taxonomy, - "PRJ1", "chr1", start, reference, alternate, rsAccession, false, false, - false, false, 1); + "PRJ1", "chr1", start, reference, alternate, rsAccession, false, false, + false, false, 1); } private DbsnpClusteredVariantEntity createRS(SubmittedVariantEntity sve, Integer alternateTaxonomy, Long start) { - Function hashingFunction = new ClusteredVariantSummaryFunction().andThen( + Function hashingFunction = new ClusteredVariantSummaryFunction().andThen( new SHA1HashingFunction()); - int taxonomyToUse = Objects.isNull(alternateTaxonomy)? sve.getTaxonomyAccession(): alternateTaxonomy; + int taxonomyToUse = Objects.isNull(alternateTaxonomy) ? sve.getTaxonomyAccession() : alternateTaxonomy; long startToUse = Objects.isNull(start) ? sve.getStart() : start; ClusteredVariant cv = new ClusteredVariant(sve.getReferenceSequenceAccession(), taxonomyToUse, - sve.getContig(), - startToUse, - new Variant(sve.getContig(), sve.getStart(), sve.getStart(), - sve.getReferenceAllele(), - sve.getAlternateAllele()).getType(), - true, null); + sve.getContig(), + startToUse, + new Variant(sve.getContig(), sve.getStart(), sve.getStart(), + sve.getReferenceAllele(), + sve.getAlternateAllele()).getType(), + true, null); String hash = hashingFunction.apply(cv); return new DbsnpClusteredVariantEntity(sve.getClusteredVariantAccession(), hash, cv); } diff --git a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/merged/MergedVariantMongoReaderTest.java b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/merged/MergedVariantMongoReaderTest.java index da5f5e750..257d5bee5 100644 --- a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/merged/MergedVariantMongoReaderTest.java +++ b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/merged/MergedVariantMongoReaderTest.java @@ -16,23 +16,19 @@ package uk.ac.ebi.eva.accession.release.batch.io.merged; -import com.lordofthejars.nosqlunit.annotation.UsingDataSet; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; -import com.mongodb.MongoClient; -import org.junit.After; -import org.junit.Before; -import org.junit.Ignore; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; +import com.mongodb.client.MongoClient; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.batch.item.ExecutionContext; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.ApplicationContext; +import org.springframework.core.io.ResourceLoader; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.ampt2d.commons.accession.core.models.EventType; import uk.ac.ebi.ampt2d.commons.accession.hashing.SHA1HashingFunction; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.MongoConfiguration; @@ -47,10 +43,11 @@ import uk.ac.ebi.eva.accession.core.model.eva.ClusteredVariantEntity; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; import uk.ac.ebi.eva.accession.core.summary.ClusteredVariantSummaryFunction; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; +import uk.ac.ebi.eva.accession.core.utils.MongoTestDataLoader; import uk.ac.ebi.eva.accession.release.collectionNames.DbsnpCollectionNames; import uk.ac.ebi.eva.accession.release.collectionNames.EvaCollectionNames; import uk.ac.ebi.eva.accession.release.test.configuration.MongoTestConfiguration; -import uk.ac.ebi.eva.accession.release.test.rule.FixSpringMongoDbRule; import uk.ac.ebi.eva.commons.core.models.pipeline.Variant; import java.util.ArrayList; @@ -63,13 +60,11 @@ import java.util.stream.Collectors; import java.util.stream.Stream; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; -import static org.springframework.data.mongodb.core.query.Criteria.where; -import static org.springframework.data.mongodb.core.query.Query.query; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import static uk.ac.ebi.eva.accession.release.batch.io.active.AccessionedVariantMongoReader.CLUSTERED_VARIANT_VALIDATED_KEY; import static uk.ac.ebi.eva.accession.release.batch.io.active.AccessionedVariantMongoReader.REMAPPED_KEY; import static uk.ac.ebi.eva.accession.release.batch.io.active.AccessionedVariantMongoReader.SUBMITTED_VARIANT_VALIDATED_KEY; @@ -78,18 +73,10 @@ import static uk.ac.ebi.eva.accession.release.batch.io.merged.MergedVariantMongoReader.MERGED_INTO_KEY; import static uk.ac.ebi.eva.accession.release.batch.io.merged.MergedVariantMongoReader.SUPPORTED_BY_EVIDENCE_KEY; -@RunWith(SpringRunner.class) +@ExtendWith(SpringExtension.class) @TestPropertySource("classpath:application.properties") -@UsingDataSet(locations = { - "/test-data/dbsnpClusteredVariantOperationEntity.json", - "/test-data/clusteredVariantOperationEntity.json", - "/test-data/dbsnpSubmittedVariantOperationEntity.json", - "/test-data/submittedVariantOperationEntity.json", // includes 1 update involving dbSNP RS merge - "/test-data/dbsnpClusteredVariantEntity.json", - "/test-data/clusteredVariantEntity.json" // includes 1 RS that was a merge target for a dbSNP RS -}) @ContextConfiguration(classes = {MongoConfiguration.class, MongoTestConfiguration.class}) -public class MergedVariantMongoReaderTest { +public class MergedVariantMongoReaderTest extends MongoTestContainerHelper { private static final String TEST_DB = "test-db"; @@ -121,40 +108,42 @@ public class MergedVariantMongoReaderTest { @Autowired private MongoTemplate mongoTemplate; - //Required by nosql-unit @Autowired - private ApplicationContext applicationContext; - - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName(TEST_DB).build()); + private ResourceLoader resourceLoader; private MergedVariantMongoReader defaultReader; private ExecutionContext executionContext; - @Before + @BeforeEach public void setUp() throws Exception { + mongoTemplate.getDb().drop(); + + MongoTestDataLoader mongoTestDataLoader = new MongoTestDataLoader(mongoTemplate, resourceLoader); + mongoTestDataLoader.load("/test-data/dbsnpClusteredVariantOperationEntity.json"); + mongoTestDataLoader.load("/test-data/clusteredVariantOperationEntity.json"); + mongoTestDataLoader.load("/test-data/dbsnpSubmittedVariantOperationEntity.json"); + mongoTestDataLoader.load("/test-data/submittedVariantOperationEntity.json"); + mongoTestDataLoader.load("/test-data/dbsnpClusteredVariantEntity.json"); + mongoTestDataLoader.load("/test-data/clusteredVariantEntity.json"); + executionContext = new ExecutionContext(); defaultReader = new MergedVariantMongoReader(ASSEMBLY, TAXONOMY, mongoClient, TEST_DB, CHUNK_SIZE, - new DbsnpCollectionNames()); + new DbsnpCollectionNames()); } - @After - public void tearDown() throws Exception { - this.mongoTemplate.findAllAndRemove(query(where("inactiveObjects.tax").in(Arrays.asList(TAX1, TAX2))), - DbsnpSubmittedVariantOperationEntity.class); - this.mongoTemplate.findAllAndRemove(query(where("inactiveObjects.tax").in(Arrays.asList(TAX1, TAX2))), - DbsnpClusteredVariantOperationEntity.class); + @AfterEach + public void tearDown() { + mongoTemplate.getDb().drop(); } @Test - public void basicRead() throws Exception { + public void basicRead() { Map variants = readIntoMap(defaultReader); assertEquals(EXPECTED_MERGED_VARIANTS, variants.size()); } - private Map readIntoMap(MergedVariantMongoReader reader) throws Exception { + private Map readIntoMap(MergedVariantMongoReader reader) { reader.open(executionContext); Map allVariants = new HashMap<>(); List variants; @@ -173,23 +162,23 @@ private String getStringId(Variant variant) { } @Test - public void checkMergedInto() throws Exception { + public void checkMergedInto() { Map variants = readIntoMap(defaultReader); assertEquals(EXPECTED_MERGED_VARIANTS, variants.size()); assertTrue(variants.get(ID_1_A) - .getSourceEntries() - .stream() - .allMatch(e -> ID_1_MERGED_INTO.equals(e.getAttribute(MERGED_INTO_KEY)))); + .getSourceEntries() + .stream() + .allMatch(e -> ID_1_MERGED_INTO.equals(e.getAttribute(MERGED_INTO_KEY)))); assertTrue(variants.get(ID_1_T) - .getSourceEntries() - .stream() - .allMatch(e -> ID_1_MERGED_INTO.equals(e.getAttribute(MERGED_INTO_KEY)))); + .getSourceEntries() + .stream() + .allMatch(e -> ID_1_MERGED_INTO.equals(e.getAttribute(MERGED_INTO_KEY)))); } @Test - public void checkAlleles() throws Exception { + public void checkAlleles() { Map variants = readIntoMap(defaultReader); assertEquals(EXPECTED_MERGED_VARIANTS, variants.size()); @@ -204,58 +193,58 @@ public void checkAlleles() throws Exception { } @Test - public void includeValidatedFlag() throws Exception { + public void includeValidatedFlag() { assertFlagEqualsInAllVariants(CLUSTERED_VARIANT_VALIDATED_KEY, false); assertFlagEqualsInAllVariants(SUBMITTED_VARIANT_VALIDATED_KEY, false); } - private void assertFlagEqualsInAllVariants(String key, boolean value) throws Exception { + private void assertFlagEqualsInAllVariants(String key, boolean value) { Map variants = readIntoMap(defaultReader); assertNotEquals(0, variants.size()); assertTrue(variants.values().stream() - .flatMap(v -> v.getSourceEntries().stream()) - .map(se -> se.getAttribute(key)) - .map(Boolean::new) - .allMatch(v -> v.equals(value))); + .flatMap(v -> v.getSourceEntries().stream()) + .map(se -> se.getAttribute(key)) + .map(Boolean::parseBoolean) + .allMatch(v -> v.equals(value))); } @Test - public void includeAssemblyMatchFlag() throws Exception { + public void includeAssemblyMatchFlag() { assertFlagEqualsInAllVariants(ASSEMBLY_MATCH_KEY, true); } @Test - public void includeAllelesMatchFlag() throws Exception { + public void includeAllelesMatchFlag() { assertFlagEqualsInAllVariants(ALLELES_MATCH_KEY, true); } @Test - public void includeEvidenceFlag() throws Exception { + public void includeEvidenceFlag() { assertFlagEqualsInAllVariants(SUPPORTED_BY_EVIDENCE_KEY, false); } @Test - public void includeRemappedFlag() throws Exception { + public void includeRemappedFlag() { MergedVariantMongoReader evaReader = new MergedVariantMongoReader(ASSEMBLY, TAXONOMY, mongoClient, TEST_DB, - CHUNK_SIZE, new EvaCollectionNames()); + CHUNK_SIZE, new EvaCollectionNames()); Map variants = readIntoMap(evaReader); assertNotEquals(0, variants.size()); List rsToLookFor = variants.values().stream().filter(v -> v.getMainId().equals("rs3000000010")) - .collect(Collectors.toList()); + .collect(Collectors.toList()); assertNotEquals(0, rsToLookFor.size()); // Both the SS involved in SVOE/dbsnpSVOE operations has the "remappedFrom" attribute assertTrue(rsToLookFor.stream().flatMap(v -> v.getSourceEntries().stream()) - .map(se -> se.getAttribute(REMAPPED_KEY)) - .map(Boolean::new) - .allMatch(v -> v.equals(true))); + .map(se -> se.getAttribute(REMAPPED_KEY)) + .map(Boolean::parseBoolean) + .allMatch(v -> v.equals(true))); rsToLookFor = variants.values().stream().filter(v -> v.getMainId().equals("rs3000000020")) - .collect(Collectors.toList()); + .collect(Collectors.toList()); assertNotEquals(0, rsToLookFor.size()); // Only one of the SS involved in SVOE/dbsnpSVOE operations has the "remappedFrom" attribute assertTrue(rsToLookFor.stream().flatMap(v -> v.getSourceEntries().stream()) - .map(se -> se.getAttribute(REMAPPED_KEY)) - .map(Boolean::new) - .allMatch(v -> v.equals(false))); + .map(se -> se.getAttribute(REMAPPED_KEY)) + .map(Boolean::parseBoolean) + .allMatch(v -> v.equals(false))); } /** @@ -265,39 +254,39 @@ public void includeRemappedFlag() throws Exception { * - 1 Update operation in dbsnpSubmittedVariantOperationEntity for ss1986084768 (Original rs881301177 was merged * into rs80393223) * - 1 Merge operation in dbsnpSubmittedVariantOperationEntity for ss1986084768 - * + *

* Even though the rs80393223 was involved in a merge operation it doesn't mean all of its associated variants were * also involved in that merge. Hence we should only list the variants that were updated, merge operations of * submitted variants should be ignored. */ @Test - public void includeOnlyMergedVariants() throws Exception { + public void includeOnlyMergedVariants() { MergedVariantMongoReader reader = new MergedVariantMongoReader("GCA_000001215.4", 7227, - mongoClient, TEST_DB, - CHUNK_SIZE, new DbsnpCollectionNames()); + mongoClient, TEST_DB, + CHUNK_SIZE, new DbsnpCollectionNames()); Map allVariants = readIntoMap(reader); assertEquals(1, allVariants.size()); assertEquals("rs881301177", allVariants.get("AE013599.5_7680720_T_").getMainId()); assertEquals("rs80393223", allVariants.get("AE013599.5_7680720_T_").getSourceEntries().iterator().next() - .getAttributes().get("CURR")); + .getAttributes().get("CURR")); } /** * This test will use a different defaultReader for assembly GCA_000002305.1 to evaluate this specific scenario: * - 2 Merge operations for Clustered Variants with the same accession (rs69314228) and mergeInto (rs68736359) but - * different chromosome or/and start + * different chromosome or/and start * - 2 Update (merge RS) operations for Submitted Variants * - 1 Update (decluster) operation for Submitted Variants - * + *

* For every Clustered Variant Operation there will be N Submitted Variant Operations but only the merged * operations with the same chromosome and start must be taking into account to build the merged VCF file. */ @Test - public void includeOnlyMergedVariantsWithSameChrAndSameStartInRsAndSs() throws Exception { + public void includeOnlyMergedVariantsWithSameChrAndSameStartInRsAndSs() { MergedVariantMongoReader reader = new MergedVariantMongoReader("GCA_000002305.1", 9796, - mongoClient, TEST_DB, - CHUNK_SIZE, new DbsnpCollectionNames()); + mongoClient, TEST_DB, + CHUNK_SIZE, new DbsnpCollectionNames()); Map allVariants = readIntoMap(reader); assertEquals(2, allVariants.size()); @@ -313,18 +302,18 @@ public void includeOnlyMergedVariantsWithSameChrAndSameStartInRsAndSs() throws E * This test will use a different defaultReader for assembly GCA_000001635.5 to evaluate this specific scenario: * - One merge operation for clustered variants (rs258660893 merged into rs244942339) * - One update operations for submitted variants but the reason is a DECLUSTER - * + *

* This happens because the decluster operations are created in the processor and the merge operations in the writer. - * + *

* So we will have some situations where there is a merge operation but the merged clustered variant does not have * any submitted variants associated because they all have been declustered (rs = null). Hence no update operation * to indicate the merge is created for submitted variants. */ @Test - public void noExceptionIfSsWereDeclustered() throws Exception { + public void noExceptionIfSsWereDeclustered() { MergedVariantMongoReader reader = new MergedVariantMongoReader("GCA_000001635.5", 10090, - mongoClient, TEST_DB, - CHUNK_SIZE, new DbsnpCollectionNames()); + mongoClient, TEST_DB, + CHUNK_SIZE, new DbsnpCollectionNames()); Map allVariants = readIntoMap(reader); assertEquals(0, allVariants.size()); } @@ -333,14 +322,15 @@ public void noExceptionIfSsWereDeclustered() throws Exception { * This test will use a different defaultReader for assembly GCA_000181335.3 to evaluate this specific scenario: * - One merge operation for clustered variants (rs782965190 merged into rs43955718) * - No submitted variants operations associated to rs782965190 - * + *

* Given that there are not merge nor decluster operations for any submitted variant associated this should * not return any records */ - public void exceptionIfRsMergedHasNoSsMergeOperations() throws Exception { + @Test + public void exceptionIfRsMergedHasNoSsMergeOperations() { MergedVariantMongoReader reader = new MergedVariantMongoReader("GCA_000181335.3", 9685, - mongoClient, TEST_DB, - CHUNK_SIZE, new DbsnpCollectionNames()); + mongoClient, TEST_DB, + CHUNK_SIZE, new DbsnpCollectionNames()); assertEquals(0, readIntoMap(reader).size()); } @@ -349,14 +339,14 @@ public void exceptionIfRsMergedHasNoSsMergeOperations() throws Exception { * - Two merge operation for clustered variants * - Two corresponding operations for its submitted variants * - Only one of the clustered variants is active (Present in dbsnpClusteredVariantEntity collection) - * + *

* Only the clustered operation associated to the active clustered variant should be returned */ @Test public void excludeMergedIntoADeprecatedRs() throws Exception { MergedVariantMongoReader reader = new MergedVariantMongoReader("GCA_000004515.3", 3847, - mongoClient, TEST_DB, - CHUNK_SIZE, new DbsnpCollectionNames()); + mongoClient, TEST_DB, + CHUNK_SIZE, new DbsnpCollectionNames()); Map allVariants = readIntoMap(reader); assertEquals(1, allVariants.size()); assertNotNull(allVariants.get("CM000851.2_2715437_G_A")); @@ -368,24 +358,24 @@ public void excludeMergedIntoADeprecatedRs() throws Exception { * and that RS ID in one location can be deprecated but active in the other one. In that case we will count the * RS ID as active because it will be present in the dbsnpClusteredVariantEntity collection for at least one * location. - * + *

* This test will use a different defaultReader for assembly GCA_000001111.1 to evaluate this specific scenario: * - Two merge operations for clustered variants to the same RS ID (rs2222->rs1111 and rs3333->rs1111). Note that * rs1111 is mapped to multiple locations (start:100 and start:200) * - Two submitted variant operations indicating the merged operations of its RS ID * - Only one clustered variant in the active collection (rs1111 start:100). This means that rs1111 start:200 has * been deprecated - * + *

* Even though the rs1111 is only active with start 100, the merge reader will also include rs1111 as * "current RS ID" for the line of rs2222 with start 200 in the merged VCF. */ - @Ignore("This test is ignored because it would fail with the current implementation but we need to be aware of" + + @Disabled("This test is ignored because it would fail with the current implementation but we need to be aware of" + "this situation (More details in javadoc)") @Test - public void rsMappedToDifferentLocationsOneDeprecated() throws Exception { + public void rsMappedToDifferentLocationsOneDeprecated() { MergedVariantMongoReader reader = new MergedVariantMongoReader("GCA_000001111.1", 1111, - mongoClient, TEST_DB, - CHUNK_SIZE, new DbsnpCollectionNames()); + mongoClient, TEST_DB, + CHUNK_SIZE, new DbsnpCollectionNames()); Map allVariants = readIntoMap(reader); //Should return only RS ID with start:100 @@ -404,14 +394,14 @@ public void rsMappedToDifferentLocationsOneDeprecated() throws Exception { * This test will use a different defaultReader for assembly GCA_000001635.4 to evaluate this specific scenario: * - One merge operation for clustered variants (rs258660892 merged into rs244942338) * - One update operation for submitted variants but in the EVA collection (clustered under a dbSNP RS). - * + *

* In this case we should not throw an exception but find the variant in the other collection. */ @Test - public void noExceptionIfSsFromDifferentCollectionUpdated() throws Exception { + public void noExceptionIfSsFromDifferentCollectionUpdated() { MergedVariantMongoReader reader = new MergedVariantMongoReader("GCA_000001635.4", 10090, - mongoClient, TEST_DB, - CHUNK_SIZE, new DbsnpCollectionNames()); + mongoClient, TEST_DB, + CHUNK_SIZE, new DbsnpCollectionNames()); Map allVariants = readIntoMap(reader); assertEquals(1, allVariants.size()); } @@ -419,20 +409,20 @@ public void noExceptionIfSsFromDifferentCollectionUpdated() throws Exception { /** * This test will use a different defaultReader for assembly GCA_002863925.1 to evaluate this specific scenario: * - One merge operation for clustered variants (dbSNP rs3091764863 merged into EVA rs393745096) - * + *

* In this case we should successfully find the merged dbSNP clustered variant. */ @Test - public void rsMergedWithRsFromDifferentCollection() throws Exception { + public void rsMergedWithRsFromDifferentCollection() { MergedVariantMongoReader reader = new MergedVariantMongoReader("GCA_002863925.1", 9796, - mongoClient, TEST_DB, - CHUNK_SIZE, new DbsnpCollectionNames()); + mongoClient, TEST_DB, + CHUNK_SIZE, new DbsnpCollectionNames()); Map allVariants = readIntoMap(reader); assertEquals(1, allVariants.size()); } @Test - public void testOnlySpecifiedTaxVariantsRead() throws Exception { + public void testOnlySpecifiedTaxVariantsRead() { // See scenario here: https://docs.google.com/spreadsheets/d/12QJT4N0-UJGTv3BtVq_gyyrVzweXd5ev2WFlnP-4MW4/edit#rangeid=169520205 DbsnpSubmittedVariantEntity ss1 = createSS(ASSEMBLY, TAX1, 1L, 1L, 100L, "C", "A"); DbsnpSubmittedVariantEntity ss2 = createSS(ASSEMBLY, TAX2, 2L, 2L, 101L, "A", "T"); @@ -443,26 +433,26 @@ public void testOnlySpecifiedTaxVariantsRead() throws Exception { this.mongoTemplate.insert(Stream.of(ss1, ss2, ss3).map(ss -> { DbsnpSubmittedVariantOperationEntity dbsnpSvoeObj = new DbsnpSubmittedVariantOperationEntity(); dbsnpSvoeObj.fill(EventType.UPDATED, ss.getAccession(), - "Original RS was merged into another RS", - Arrays.asList(new DbsnpSubmittedVariantInactiveEntity(ss))); + "Original RS was merged into another RS", + Arrays.asList(new DbsnpSubmittedVariantInactiveEntity(ss))); return dbsnpSvoeObj; }).collect(Collectors.toList()), DbsnpSubmittedVariantOperationEntity.class); this.mongoTemplate.insert(Stream.of(rs1, rs2, rs3).map(rs -> { DbsnpClusteredVariantOperationEntity dbsnpCvoeObj = new DbsnpClusteredVariantOperationEntity(); - dbsnpCvoeObj.fill( EventType.MERGED, rs.getAccession(), rs.getAccession() + 10, - "Original RS was merged into another RS", - Arrays.asList(new DbsnpClusteredVariantInactiveEntity(rs))); + dbsnpCvoeObj.fill(EventType.MERGED, rs.getAccession(), rs.getAccession() + 10, + "Original RS was merged into another RS", + Arrays.asList(new DbsnpClusteredVariantInactiveEntity(rs))); return dbsnpCvoeObj; }).collect(Collectors.toList()), DbsnpClusteredVariantOperationEntity.class); // If we don't insert target CVEs of the merges, the merge reader will fail // because such merged RS will be categorized as merged-then-deprecated this.mongoTemplate.insert(Stream.of(rs1, rs2, rs3).map(rs -> new ClusteredVariantEntity( - rs.getAccession() + 10, rs.getHashedMessage(), rs.getModel())) - .collect(Collectors.toList()), DbsnpClusteredVariantEntity.class); + rs.getAccession() + 10, rs.getHashedMessage(), rs.getModel())) + .collect(Collectors.toList()), DbsnpClusteredVariantEntity.class); // Test records returned when the mongo reader is given TAX1 - MergedVariantMongoReader reader = new MergedVariantMongoReader(ASSEMBLY, TAX1, mongoClient, TEST_DB, - CHUNK_SIZE, new DbsnpCollectionNames()); + MergedVariantMongoReader reader = new MergedVariantMongoReader(ASSEMBLY, TAX1, mongoClient, TEST_DB, + CHUNK_SIZE, new DbsnpCollectionNames()); reader.open(new ExecutionContext()); List mergedVariants = new ArrayList<>(readIntoMap(reader).values()); assertEquals(1, mergedVariants.size()); @@ -472,7 +462,7 @@ public void testOnlySpecifiedTaxVariantsRead() throws Exception { // Note rs3 is returned even though in Mongo it was originally issued in TAX1 // This is because, before the merge, there was a SS record in TAX2 that was clustered under rs3 reader = new MergedVariantMongoReader(ASSEMBLY, TAX2, mongoClient, TEST_DB, CHUNK_SIZE, - new DbsnpCollectionNames()); + new DbsnpCollectionNames()); reader.open(new ExecutionContext()); mergedVariants = new ArrayList<>(readIntoMap(reader).values()); assertEquals(2, mergedVariants.size()); @@ -483,21 +473,21 @@ public void testOnlySpecifiedTaxVariantsRead() throws Exception { private DbsnpSubmittedVariantEntity createSS(String assembly, int taxonomy, Long ssAccession, Long rsAccession, Long start, String reference, String alternate) { return new DbsnpSubmittedVariantEntity(ssAccession, "hash" + ssAccession, assembly, taxonomy, - "PRJ1", "chr1", start, reference, alternate, rsAccession, false, false, - false, false, 1); + "PRJ1", "chr1", start, reference, alternate, rsAccession, false, false, + false, false, 1); } private DbsnpClusteredVariantEntity createRS(SubmittedVariantEntity sve, Integer alternateTaxonomy) { - Function hashingFunction = new ClusteredVariantSummaryFunction().andThen( + Function hashingFunction = new ClusteredVariantSummaryFunction().andThen( new SHA1HashingFunction()); - int taxonomyToUse = Objects.isNull(alternateTaxonomy)? sve.getTaxonomyAccession(): alternateTaxonomy; + int taxonomyToUse = Objects.isNull(alternateTaxonomy) ? sve.getTaxonomyAccession() : alternateTaxonomy; ClusteredVariant cv = new ClusteredVariant(sve.getReferenceSequenceAccession(), taxonomyToUse, - sve.getContig(), - sve.getStart(), - new Variant(sve.getContig(), sve.getStart(), sve.getStart(), - sve.getReferenceAllele(), - sve.getAlternateAllele()).getType(), - true, null); + sve.getContig(), + sve.getStart(), + new Variant(sve.getContig(), sve.getStart(), sve.getStart(), + sve.getReferenceAllele(), + sve.getAlternateAllele()).getType(), + true, null); String hash = hashingFunction.apply(cv); return new DbsnpClusteredVariantEntity(sve.getClusteredVariantAccession(), hash, cv); } diff --git a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/merged_deprecated/MergedDeprecatedVariantAccessionWriterTest.java b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/merged_deprecated/MergedDeprecatedVariantAccessionWriterTest.java index 432af7f40..b9193fad0 100644 --- a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/merged_deprecated/MergedDeprecatedVariantAccessionWriterTest.java +++ b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/merged_deprecated/MergedDeprecatedVariantAccessionWriterTest.java @@ -15,33 +15,31 @@ */ package uk.ac.ebi.eva.accession.release.batch.io.merged_deprecated; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.batch.item.Chunk; import uk.ac.ebi.ampt2d.commons.accession.core.models.EventType; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpClusteredVariantOperationEntity; +import uk.ac.ebi.eva.accession.core.utils.PipelineTemporaryFolderUtil; import java.io.BufferedReader; import java.io.File; import java.io.FileReader; import java.io.IOException; -import java.util.Arrays; import java.util.Optional; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; public class MergedDeprecatedVariantAccessionWriterTest { private MergedDeprecatedVariantAccessionWriter writer; - @Rule - public TemporaryFolder temporaryFolderRule = new TemporaryFolder(); + public PipelineTemporaryFolderUtil temporaryFolderUtil = new PipelineTemporaryFolderUtil(); - @Before + @BeforeEach public void setUp() throws Exception { - File output = temporaryFolderRule.newFile(); + File output = temporaryFolderUtil.newFile(); writer = new MergedDeprecatedVariantAccessionWriter(output.toPath()); } @@ -55,7 +53,7 @@ public void writeLines() throws Exception { variant3.fill(EventType.MERGED, 3L, 13L, "Reason", null); writer.open(null); - writer.write(Arrays.asList(variant1, variant2, variant3)); + writer.write(Chunk.of(variant1, variant2, variant3)); writer.close(); assertEquals(3, numberOfLines(writer.getOutput())); @@ -73,7 +71,7 @@ public void twoColumns() throws Exception { long accessionIdDestiny = 11L; variant1.fill(EventType.MERGED, accessionIdOrigin, accessionIdDestiny, "Reason", null); writer.open(null); - writer.write(Arrays.asList(variant1)); + writer.write(Chunk.of(variant1)); writer.close(); Optional line = new BufferedReader(new FileReader( diff --git a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/merged_deprecated/MergedDeprecatedVariantMongoReaderTest.java b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/merged_deprecated/MergedDeprecatedVariantMongoReaderTest.java index f151e0dfe..ef211280c 100644 --- a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/merged_deprecated/MergedDeprecatedVariantMongoReaderTest.java +++ b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/merged_deprecated/MergedDeprecatedVariantMongoReaderTest.java @@ -16,44 +16,35 @@ package uk.ac.ebi.eva.accession.release.batch.io.merged_deprecated; -import com.lordofthejars.nosqlunit.annotation.UsingDataSet; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; -import com.mongodb.MongoClient; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; +import com.mongodb.client.MongoClient; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.batch.item.ExecutionContext; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.ApplicationContext; +import org.springframework.core.io.ResourceLoader; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.ampt2d.commons.accession.core.models.EventType; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.MongoConfiguration; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpClusteredVariantOperationEntity; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; +import uk.ac.ebi.eva.accession.core.utils.MongoTestDataLoader; import uk.ac.ebi.eva.accession.release.collectionNames.DbsnpCollectionNames; import uk.ac.ebi.eva.accession.release.test.configuration.MongoTestConfiguration; -import uk.ac.ebi.eva.accession.release.test.rule.FixSpringMongoDbRule; import java.util.ArrayList; import java.util.List; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; -@RunWith(SpringRunner.class) +@ExtendWith(SpringExtension.class) @TestPropertySource("classpath:application.properties") -@UsingDataSet(locations = { - "/test-data/dbsnpClusteredVariantOperationEntity.json", - "/test-data/dbsnpSubmittedVariantOperationEntity.json", - "/test-data/dbsnpClusteredVariantEntity.json", - "/test-data/dbsnpSubmittedVariantEntity.json" -}) @ContextConfiguration(classes = {MongoConfiguration.class, MongoTestConfiguration.class}) -public class MergedDeprecatedVariantMongoReaderTest { +public class MergedDeprecatedVariantMongoReaderTest extends MongoTestContainerHelper { private static final String TEST_DB = "test-db"; @@ -66,30 +57,35 @@ public class MergedDeprecatedVariantMongoReaderTest { @Autowired private MongoTemplate mongoTemplate; - @Autowired - private MongoClient mongoClient; - //Required by nosql-unit @Autowired - private ApplicationContext applicationContext; + private ResourceLoader resourceLoader; - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName(TEST_DB).build()); + @Autowired + private MongoClient mongoClient; private MergedDeprecatedVariantMongoReader reader; - @Before + @BeforeEach public void setUp() { + mongoTemplate.getDb().drop(); + + MongoTestDataLoader mongoTestDataLoader = new MongoTestDataLoader(mongoTemplate, resourceLoader); + mongoTestDataLoader.load("/test-data/dbsnpClusteredVariantOperationEntity.json"); + mongoTestDataLoader.load("/test-data/dbsnpSubmittedVariantOperationEntity.json"); + mongoTestDataLoader.load("/test-data/dbsnpClusteredVariantEntity.json"); + mongoTestDataLoader.load("/test-data/dbsnpSubmittedVariantEntity.json"); + ExecutionContext executionContext = new ExecutionContext(); reader = new DbsnpMergedDeprecatedVariantMongoReader(ASSEMBLY, TAXONOMY, mongoClient, TEST_DB, - mongoTemplate.getConverter(), CHUNK_SIZE, - new DbsnpCollectionNames()); + mongoTemplate.getConverter(), CHUNK_SIZE, + new DbsnpCollectionNames()); reader.open(executionContext); } - @After + @AfterEach public void tearDown() { + mongoTemplate.getDb().drop(); reader.close(); } diff --git a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/processors/ContextNucleotideAdditionProcessorTest.java b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/processors/ContextNucleotideAdditionProcessorTest.java index f88ce0cfb..d304c7d00 100644 --- a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/processors/ContextNucleotideAdditionProcessorTest.java +++ b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/processors/ContextNucleotideAdditionProcessorTest.java @@ -17,13 +17,12 @@ */ package uk.ac.ebi.eva.accession.release.batch.processors; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; - +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import uk.ac.ebi.eva.accession.core.batch.io.FastaSynonymSequenceReader; import uk.ac.ebi.eva.accession.core.contig.ContigMapping; import uk.ac.ebi.eva.accession.core.contig.ContigSynonyms; -import uk.ac.ebi.eva.accession.core.batch.io.FastaSynonymSequenceReader; import uk.ac.ebi.eva.accession.core.exceptions.PositionOutsideOfContigException; import uk.ac.ebi.eva.commons.core.models.IVariant; import uk.ac.ebi.eva.commons.core.models.pipeline.Variant; @@ -33,9 +32,11 @@ import java.util.Arrays; import java.util.Collections; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.fail; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; public class ContextNucleotideAdditionProcessorTest { @@ -53,18 +54,18 @@ public class ContextNucleotideAdditionProcessorTest { private static ContextNucleotideAdditionProcessor contextNucleotideAdditionProcessor; - @BeforeClass + @BeforeAll public static void setUpClass() throws Exception { Path fastaPath = Paths.get("../eva-accession-core/src/test/resources/input-files/fasta/Gallus_gallus-5.0.test.fa"); ContigMapping contigMapping = new ContigMapping( Arrays.asList(new ContigSynonyms(CONTIG, "", "", "", "", "", true), - new ContigSynonyms(SCAFFOLD, "", "", SCAFFOLD_GENBANK_IN_FASTA, "", "", true), - new ContigSynonyms(MISSING_IN_FASTA, "", "", "", "", "", true))); + new ContigSynonyms(SCAFFOLD, "", "", SCAFFOLD_GENBANK_IN_FASTA, "", "", true), + new ContigSynonyms(MISSING_IN_FASTA, "", "", "", "", "", true))); fastaSynonymSequenceReader = new FastaSynonymSequenceReader(contigMapping, fastaPath); contextNucleotideAdditionProcessor = new ContextNucleotideAdditionProcessor(fastaSynonymSequenceReader); } - @AfterClass + @AfterAll public static void tearDownClass() throws Exception { fastaSynonymSequenceReader.close(); } @@ -198,15 +199,18 @@ public void addContextBaseUsingSynonymContig() throws Exception { assertEquals("TA", processedVariant.getAlternate()); } - @Test(expected = IllegalArgumentException.class) - public void contigNotFound() throws Exception { + @Test + void contigNotFound() { Variant variant1 = new Variant(MISSING_IN_FASTA, 10, 10, "", "A"); - try { - contextNucleotideAdditionProcessor.process(variant1); - } catch (PositionOutsideOfContigException wrongException) { - fail("The exception (" + wrongException.getClass().getSimpleName() - + ") is wrong because the variant doesn't have a position outside of chromosome. The correct " - + "exception should be that the contig is not present in the fasta"); + + Exception exception = assertThrows(Exception.class, () -> contextNucleotideAdditionProcessor.process(variant1)); + + if (exception instanceof PositionOutsideOfContigException) { + fail("The exception (" + exception.getClass().getSimpleName() + + ") is wrong because the variant doesn't have a position outside of chromosome. The correct " + + "exception should be that the contig is not present in the fasta"); } + + assertTrue(exception instanceof IllegalArgumentException); } } diff --git a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/processors/ExcludeInvalidVariantsProcessorTest.java b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/processors/ExcludeInvalidVariantsProcessorTest.java index a16b5cecf..4c6153bf6 100644 --- a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/processors/ExcludeInvalidVariantsProcessorTest.java +++ b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/processors/ExcludeInvalidVariantsProcessorTest.java @@ -16,16 +16,13 @@ package uk.ac.ebi.eva.accession.release.batch.processors; -import org.junit.BeforeClass; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; - +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; import uk.ac.ebi.eva.commons.core.models.pipeline.Variant; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; public class ExcludeInvalidVariantsProcessorTest { @@ -49,10 +46,7 @@ public class ExcludeInvalidVariantsProcessorTest { private static ExcludeInvalidVariantsProcessor processor; - @Rule - public ExpectedException thrown = ExpectedException.none(); - - @BeforeClass + @BeforeAll public static void setUp() { processor = new ExcludeInvalidVariantsProcessor(); } diff --git a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/processors/NamedVariantProcessorTest.java b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/processors/NamedVariantProcessorTest.java index 4e9a0d866..ab29a9a1c 100644 --- a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/processors/NamedVariantProcessorTest.java +++ b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/processors/NamedVariantProcessorTest.java @@ -16,7 +16,7 @@ package uk.ac.ebi.eva.accession.release.batch.processors; -import org.junit.Test; +import org.junit.jupiter.api.Test; import uk.ac.ebi.eva.commons.core.models.IVariant; import uk.ac.ebi.eva.commons.core.models.VariantType; @@ -24,8 +24,9 @@ import uk.ac.ebi.eva.commons.core.models.pipeline.Variant; import uk.ac.ebi.eva.commons.core.models.pipeline.VariantSourceEntry; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import static uk.ac.ebi.eva.accession.release.batch.io.VariantMongoAggregationReader.ALLELES_MATCH_KEY; import static uk.ac.ebi.eva.accession.release.batch.io.VariantMongoAggregationReader.ASSEMBLY_MATCH_KEY; import static uk.ac.ebi.eva.accession.release.batch.io.VariantMongoAggregationReader.CLUSTERED_VARIANT_VALIDATED_KEY; @@ -72,7 +73,7 @@ private void assertNamedVariant(String reference, String alternate) throws Excep private Variant buildVariant(String chr, int start, String reference, String alternate, String sequenceOntologyTerm, String... studies) { return buildVariant(chr, start, reference, alternate, sequenceOntologyTerm, false, false, true, true, true, - studies); + studies); } private Variant buildVariant(String chr, int start, String reference, String alternate, @@ -109,9 +110,9 @@ public void processNamedDeletionWithEmptyAlternate() throws Exception { assertNamedVariant("(1190 BP DEL)", ""); } - @Test(expected = IllegalArgumentException.class) - public void throwIfBothAllelesAreNamed() throws Exception { - assertNamedVariant("(1190 BP DEL)", "(1190 BP DEL)"); + @Test + public void throwIfBothAllelesAreNamed() { + assertThrows(IllegalArgumentException.class, () -> assertNamedVariant("(1190 BP DEL)", "(1190 BP DEL)")); } @Test @@ -153,7 +154,7 @@ public void swapSymbolicReference() throws Exception { } /** - * Taken from EVA-1220. Some of these won't even go through the release in VCF format, but test them just in case + * Taken from EVA-1220. Some of these won't even go through the release in VCF format, but test them just in case */ @Test public void handleStrangeAlleles() throws Exception { diff --git a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/processors/VariantToVariantContextProcessorTest.java b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/processors/VariantToVariantContextProcessorTest.java index d8f17bd0e..210aa9fa7 100644 --- a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/processors/VariantToVariantContextProcessorTest.java +++ b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/processors/VariantToVariantContextProcessorTest.java @@ -19,11 +19,8 @@ import htsjdk.variant.variantcontext.Allele; import htsjdk.variant.variantcontext.VariantContext; import org.assertj.core.util.Sets; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; - +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import uk.ac.ebi.eva.accession.core.contig.ContigMapping; import uk.ac.ebi.eva.accession.core.contig.ContigNaming; import uk.ac.ebi.eva.accession.core.contig.ContigSynonyms; @@ -32,8 +29,9 @@ import java.util.Collections; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; public class VariantToVariantContextProcessorTest { @@ -61,10 +59,7 @@ public class VariantToVariantContextProcessorTest { private VariantToVariantContextProcessor variantConverter; - @Rule - public ExpectedException expectedException = ExpectedException.none(); - - @Before + @BeforeEach public void setUp() throws Exception { ContigMapping contigMapping = new ContigMapping(Collections.singletonList( new ContigSynonyms(SEQUENCE_NAME_1, "A", "A", GENBANK_ACCESSION_1, "A", "A", true))); @@ -72,7 +67,7 @@ public void setUp() throws Exception { } @Test - public void singleStudySNV() throws Exception { + public void singleStudySNV() { Variant variant = buildVariant(GENBANK_ACCESSION_1, 1000, "C", "A", SNP_SEQUENCE_ONTOLOGY, STUDY_1); VariantContext variantContext = variantConverter.process(variant); assertVariantContext(variantContext, SEQUENCE_NAME_1, 1000, 1000, ID, "C", "A", SNP_SEQUENCE_ONTOLOGY, STUDY_1); @@ -100,7 +95,7 @@ private void assertVariantContext(VariantContext variantContext, String expected assertEquals(expectedEnd, variantContext.getEnd()); assertEquals(Allele.create(expectedReference, true), variantContext.getReference()); assertEquals(Collections.singletonList(Allele.create(expectedAlternate, false)), - variantContext.getAlternateAlleles()); + variantContext.getAlternateAlleles()); assertEquals(expectedId, variantContext.getID()); assertTrue(variantContext.getFilters().isEmpty()); assertEquals(2, variantContext.getCommonInfo().getAttributes().size()); @@ -115,61 +110,59 @@ private void assertVariantContext(VariantContext variantContext, String expected } @Test - public void throwsIfAllelesAreEmpty() throws Exception { + public void throwsIfAllelesAreEmpty() { Variant variant = buildVariant(GENBANK_ACCESSION_1, 1100, "", "G", SNP_SEQUENCE_ONTOLOGY, STUDY_1); - expectedException.expect(IllegalArgumentException.class); - variantConverter.process(variant); + assertThrows(IllegalArgumentException.class, () -> variantConverter.process(variant)); } @Test - public void singleStudySingleNucleotideInsertion() throws Exception { + public void singleStudySingleNucleotideInsertion() { Variant variant = buildVariant(GENBANK_ACCESSION_1, 1100, "T", "TG", SNP_SEQUENCE_ONTOLOGY, STUDY_1); VariantContext variantContext = variantConverter.process(variant); assertVariantContext(variantContext, SEQUENCE_NAME_1, 1100, 1100, ID, "T", "TG", SNP_SEQUENCE_ONTOLOGY, - STUDY_1); + STUDY_1); } @Test - public void singleStudySeveralNucleotidesInsertion() throws Exception { + public void singleStudySeveralNucleotidesInsertion() { Variant variant = buildVariant(GENBANK_ACCESSION_1, 1100, "T", "TGA", INSERTION_SEQUENCE_ONTOLOGY, STUDY_1); VariantContext variantContext = variantConverter.process(variant); assertVariantContext(variantContext, SEQUENCE_NAME_1, 1100, 1100, ID, "T", "TGA", INSERTION_SEQUENCE_ONTOLOGY, - STUDY_1); + STUDY_1); } @Test - public void singleStudySingleNucleotideDeletion() throws Exception { + public void singleStudySingleNucleotideDeletion() { Variant variant = buildVariant(GENBANK_ACCESSION_1, 1100, "TA", "T", DELETION_SEQUENCE_ONTOLOGY, STUDY_1); VariantContext variantContext = variantConverter.process(variant); assertVariantContext(variantContext, SEQUENCE_NAME_1, 1100, 1101, ID, "TA", "T", DELETION_SEQUENCE_ONTOLOGY, - STUDY_1); + STUDY_1); } @Test - public void singleStudySeveralNucleotidesDeletion() throws Exception { + public void singleStudySeveralNucleotidesDeletion() { Variant variant = buildVariant(GENBANK_ACCESSION_1, 1100, "TAG", "T", DELETION_SEQUENCE_ONTOLOGY, STUDY_1); VariantContext variantContext = variantConverter.process(variant); assertVariantContext(variantContext, SEQUENCE_NAME_1, 1100, 1102, ID, "TAG", "T", DELETION_SEQUENCE_ONTOLOGY, - STUDY_1); + STUDY_1); } @Test - public void singleStudyMultiAllelicVariant() throws Exception { + public void singleStudyMultiAllelicVariant() { Variant variant = buildVariant(GENBANK_ACCESSION_1, 1100, "C", "A,T", SNP_SEQUENCE_ONTOLOGY, STUDY_1); - expectedException.expect(IllegalArgumentException.class); - variantConverter.process(variant); + assertThrows(IllegalArgumentException.class, () -> variantConverter.process(variant)); } @Test - public void singleNucleotideInsertionInPosition1() throws Exception { + public void singleNucleotideInsertionInPosition1() { Variant variant = buildVariant(GENBANK_ACCESSION_1, 1, "A", "TA", INSERTION_SEQUENCE_ONTOLOGY, STUDY_1); VariantContext variantContext = variantConverter.process(variant); assertVariantContext(variantContext, SEQUENCE_NAME_1, 1, 1, ID, "A", "TA", INSERTION_SEQUENCE_ONTOLOGY, - STUDY_1); + STUDY_1); } @Test - public void singleNucleotideDeletionInPosition1() throws Exception { + public void singleNucleotideDeletionInPosition1() { Variant variant = buildVariant(GENBANK_ACCESSION_1, 1, "AT", "T", DELETION_SEQUENCE_ONTOLOGY, STUDY_1); VariantContext variantContext = variantConverter.process(variant); assertVariantContext(variantContext, SEQUENCE_NAME_1, 1, 2, ID, "AT", "T", DELETION_SEQUENCE_ONTOLOGY, STUDY_1); @@ -177,23 +170,23 @@ public void singleNucleotideDeletionInPosition1() throws Exception { @Test - public void severalNucleotidesInsertionInPosition1() throws Exception { + public void severalNucleotidesInsertionInPosition1() { Variant variant = buildVariant(GENBANK_ACCESSION_1, 1, "A", "GGTA", INSERTION_SEQUENCE_ONTOLOGY, STUDY_1); VariantContext variantContext = variantConverter.process(variant); assertVariantContext(variantContext, SEQUENCE_NAME_1, 1, 1, ID, "A", "GGTA", INSERTION_SEQUENCE_ONTOLOGY, - STUDY_1); + STUDY_1); } @Test - public void severalNucleotidesDeletionInPosition1() throws Exception { + public void severalNucleotidesDeletionInPosition1() { Variant variant = buildVariant(GENBANK_ACCESSION_1, 1, "ATTG", "G", DELETION_SEQUENCE_ONTOLOGY, STUDY_1); VariantContext variantContext = variantConverter.process(variant); assertVariantContext(variantContext, SEQUENCE_NAME_1, 1, 4, ID, "ATTG", "G", DELETION_SEQUENCE_ONTOLOGY, - STUDY_1); + STUDY_1); } @Test - public void twoStudiesSingleVariant() throws Exception { + public void twoStudiesSingleVariant() { Variant variant = buildVariant(GENBANK_ACCESSION_1, 1000, "T", "G", SNP_SEQUENCE_ONTOLOGY, STUDY_1, STUDY_2); // process variant @@ -201,7 +194,7 @@ public void twoStudiesSingleVariant() throws Exception { // check processed variant assertVariantContext(variantContext, SEQUENCE_NAME_1, 1000, 1000, ID, "T", "G", SNP_SEQUENCE_ONTOLOGY, STUDY_1, - STUDY_2); + STUDY_2); } } diff --git a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/configuration/batch/steps/DumpRSAccessionsTest.java b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/configuration/batch/steps/DumpRSAccessionsTest.java index 2d3cb276e..5308c0b55 100644 --- a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/configuration/batch/steps/DumpRSAccessionsTest.java +++ b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/configuration/batch/steps/DumpRSAccessionsTest.java @@ -16,24 +16,19 @@ package uk.ac.ebi.eva.accession.release.configuration.batch.steps; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; -import com.mongodb.MongoClient; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.batch.core.Job; import org.springframework.batch.test.JobLauncherTestUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.context.ApplicationContext; import org.springframework.context.annotation.Bean; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.ampt2d.commons.accession.core.models.EventType; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpClusteredVariantEntity; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpClusteredVariantInactiveEntity; @@ -41,9 +36,10 @@ import uk.ac.ebi.eva.accession.core.model.eva.ClusteredVariantEntity; import uk.ac.ebi.eva.accession.core.model.eva.ClusteredVariantInactiveEntity; import uk.ac.ebi.eva.accession.core.model.eva.ClusteredVariantOperationEntity; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; +import uk.ac.ebi.eva.accession.release.test.configuration.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.accession.release.test.configuration.MongoTestConfiguration; import uk.ac.ebi.eva.accession.release.test.configuration.ReleaseFromDBTestConfiguration; -import uk.ac.ebi.eva.accession.release.test.rule.FixSpringMongoDbRule; import uk.ac.ebi.eva.commons.core.models.VariantType; import java.io.IOException; @@ -55,17 +51,17 @@ import java.util.Set; import java.util.stream.Collectors; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import static uk.ac.ebi.eva.accession.release.configuration.BeanNames.DUMP_ACTIVE_ACCESSIONS_JOB; import static uk.ac.ebi.eva.accession.release.configuration.BeanNames.DUMP_MERGED_AND_DEPRECATED_ACCESSIONS_JOB; import static uk.ac.ebi.eva.accession.release.test.configuration.ReleaseFromDBTestConfiguration.TEST_DUMP_ACTIVE_ACCESSIONS_JOB; import static uk.ac.ebi.eva.accession.release.test.configuration.ReleaseFromDBTestConfiguration.TEST_DUMP_MERGED_AND_DEPRECATED_ACCESSIONS_JOB; -@RunWith(SpringRunner.class) -@ContextConfiguration(classes = {MongoTestConfiguration.class, ReleaseFromDBTestConfiguration.class}) +@ExtendWith(SpringExtension.class) +@ContextConfiguration(classes = {MongoTestConfiguration.class, ReleaseFromDBTestConfiguration.class, + BatchJobRepositoryTestConfiguration.class}) @TestPropertySource("classpath:dump-rs-accession-test.properties") -public class DumpRSAccessionsTest { - private static final String TEST_DB = "test-db"; +public class DumpRSAccessionsTest extends MongoTestContainerHelper { private static final String accDumpFile = "src/test/resources/accDumpFile.csv"; @Autowired @@ -76,29 +72,18 @@ public class DumpRSAccessionsTest { @Qualifier(TEST_DUMP_MERGED_AND_DEPRECATED_ACCESSIONS_JOB) private JobLauncherTestUtils jobLauncherDumpMergedAndDeprecatedAccessions; - @Autowired - private MongoClient mongoClient; - @Autowired private MongoTemplate mongoTemplate; - //Required by nosql-unit - @Autowired - private ApplicationContext applicationContext; - - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName(TEST_DB).build()); - - @Before + @BeforeEach public void setUp() throws IOException { - this.mongoClient.dropDatabase(TEST_DB); + mongoTemplate.getDb().drop(); Files.deleteIfExists(Paths.get(accDumpFile)); } - @After + @AfterEach public void tearDown() throws IOException { - this.mongoClient.dropDatabase(TEST_DB); + mongoTemplate.getDb().drop(); Files.deleteIfExists(Paths.get(accDumpFile)); } diff --git a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/configuration/batch/steps/RSAccessionsReleaseFromDBTest.java b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/configuration/batch/steps/RSAccessionsReleaseFromDBTest.java index 4ae5dd4a3..527f11039 100644 --- a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/configuration/batch/steps/RSAccessionsReleaseFromDBTest.java +++ b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/configuration/batch/steps/RSAccessionsReleaseFromDBTest.java @@ -16,24 +16,19 @@ package uk.ac.ebi.eva.accession.release.configuration.batch.steps; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; -import com.mongodb.MongoClient; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.batch.core.BatchStatus; import org.springframework.batch.core.JobExecution; import org.springframework.batch.test.JobLauncherTestUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.context.ApplicationContext; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.ampt2d.commons.accession.core.models.EventType; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpClusteredVariantEntity; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpSubmittedVariantEntity; @@ -43,11 +38,12 @@ import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantInactiveEntity; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantOperationEntity; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.release.parameters.InputParameters; import uk.ac.ebi.eva.accession.release.parameters.ReportPathResolver; +import uk.ac.ebi.eva.accession.release.test.configuration.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.accession.release.test.configuration.MongoTestConfiguration; import uk.ac.ebi.eva.accession.release.test.configuration.ReleaseFromDBTestConfiguration; -import uk.ac.ebi.eva.accession.release.test.rule.FixSpringMongoDbRule; import uk.ac.ebi.eva.commons.core.models.VariantType; import uk.ac.ebi.eva.commons.core.utils.FileUtils; @@ -70,15 +66,15 @@ import java.util.Set; import java.util.stream.Collectors; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import static uk.ac.ebi.eva.accession.release.test.configuration.ReleaseFromDBTestConfiguration.TEST_RELEASE_ACTIVE_ACCESSIONS_JOB; import static uk.ac.ebi.eva.accession.release.test.configuration.ReleaseFromDBTestConfiguration.TEST_RELEASE_MERGED_AND_DEPRECATED_ACCESSIONS_JOB; -@RunWith(SpringRunner.class) -@ContextConfiguration(classes = {MongoTestConfiguration.class, ReleaseFromDBTestConfiguration.class}) +@ExtendWith(SpringExtension.class) +@ContextConfiguration(classes = {MongoTestConfiguration.class, ReleaseFromDBTestConfiguration.class, + BatchJobRepositoryTestConfiguration.class}) @TestPropertySource("classpath:dump-rs-accession-test.properties") -public class RSAccessionsReleaseFromDBTest { - private static final String TEST_DB = "test-db"; +public class RSAccessionsReleaseFromDBTest extends MongoTestContainerHelper { private static final String testRunDir = "src/test/resources/release-test-run"; private static final String rsAccFile = "src/test/resources/release-test-run/rsAccFile.csv"; @@ -93,32 +89,21 @@ public class RSAccessionsReleaseFromDBTest { @Qualifier(TEST_RELEASE_MERGED_AND_DEPRECATED_ACCESSIONS_JOB) private JobLauncherTestUtils jobLauncherReleaseMergedAndDeprecatedAccessions; - @Autowired - private MongoClient mongoClient; - @Autowired private MongoTemplate mongoTemplate; - //Required by nosql-unit - @Autowired - private ApplicationContext applicationContext; - - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName(TEST_DB).build()); - - @Before + @BeforeEach public void setUp() throws IOException { - this.mongoClient.dropDatabase(TEST_DB); + mongoTemplate.getDb().drop(); if (Files.exists(Paths.get(testRunDir))) { deleteDirectory(Paths.get(testRunDir)); } Files.createDirectories(Paths.get(testRunDir)); } - @After + @AfterEach public void tearDown() throws IOException { - this.mongoClient.dropDatabase(TEST_DB); + mongoTemplate.getDb().drop(); if (Files.exists(Paths.get(testRunDir))) { deleteDirectory(Paths.get(testRunDir)); } diff --git a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/test/configuration/BatchJobRepositoryTestConfiguration.java b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/test/configuration/BatchJobRepositoryTestConfiguration.java new file mode 100644 index 000000000..a4ef0782d --- /dev/null +++ b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/test/configuration/BatchJobRepositoryTestConfiguration.java @@ -0,0 +1,22 @@ +package uk.ac.ebi.eva.accession.release.test.configuration; + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Primary; +import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseBuilder; +import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseType; + +import javax.sql.DataSource; + +@Configuration +public class BatchJobRepositoryTestConfiguration { + + @Bean + @Primary + public DataSource dataSource() { + return new EmbeddedDatabaseBuilder() + .setType(EmbeddedDatabaseType.HSQL) + .addScript("org/springframework/batch/core/schema-hsqldb.sql") + .build(); + } +} \ No newline at end of file diff --git a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/test/configuration/MongoTestConfiguration.java b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/test/configuration/MongoTestConfiguration.java index 9c5fff6ca..3051afd22 100644 --- a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/test/configuration/MongoTestConfiguration.java +++ b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/test/configuration/MongoTestConfiguration.java @@ -17,12 +17,30 @@ */ package uk.ac.ebi.eva.accession.release.test.configuration; +import com.mongodb.ReadPreference; +import com.mongodb.WriteConcern; +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoClients; +import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.domain.EntityScan; import org.springframework.boot.test.autoconfigure.data.mongo.AutoConfigureDataMongo; +import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Primary; +import org.springframework.data.mongodb.MongoDatabaseFactory; import org.springframework.data.mongodb.config.EnableMongoAuditing; +import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.data.mongodb.core.SimpleMongoClientDatabaseFactory; +import org.springframework.data.mongodb.core.WriteResultChecking; +import org.springframework.data.mongodb.core.convert.DefaultDbRefResolver; +import org.springframework.data.mongodb.core.convert.DefaultMongoTypeMapper; +import org.springframework.data.mongodb.core.convert.MappingMongoConverter; +import org.springframework.data.mongodb.core.convert.MongoCustomConversions; +import org.springframework.data.mongodb.core.mapping.MongoMappingContext; import org.springframework.data.mongodb.repository.config.EnableMongoRepositories; +import java.util.Collections; + @Configuration @EntityScan(basePackages = {"uk.ac.ebi.eva.accession.core.persistence"}) @EnableMongoRepositories(basePackages = { @@ -32,4 +50,56 @@ @AutoConfigureDataMongo public class MongoTestConfiguration { + @Value("${spring.data.mongodb.uri}") + private String mongoUri; + + @Value("${spring.data.mongodb.database}") + private String database; + + @Value("${mongodb.read-preference:primary}") + private String readPreference; + + @Bean + @Primary + public MongoClient mongoClient() { + return MongoClients.create(mongoUri); + } + + @Bean("primaryFactory") + @Primary + public MongoDatabaseFactory mongoDbFactory() { + return new SimpleMongoClientDatabaseFactory(mongoClient(), database); + } + + @Bean + @Primary + public MongoCustomConversions mongoCustomConversions() { + return new MongoCustomConversions(Collections.emptyList()); + } + + @Bean + @Primary + public MongoMappingContext mongoMappingContext(MongoCustomConversions conversions) { + MongoMappingContext context = new MongoMappingContext(); + context.setSimpleTypeHolder(conversions.getSimpleTypeHolder()); + context.setAutoIndexCreation(false); + return context; + } + + @Bean + @Primary + public MongoTemplate mongoTemplate(MongoMappingContext mongoMappingContext, + MongoCustomConversions conversions) { + MappingMongoConverter converter = new MappingMongoConverter( + new DefaultDbRefResolver(mongoDbFactory()), mongoMappingContext); + converter.setCustomConversions(conversions); + converter.setTypeMapper(new DefaultMongoTypeMapper(null)); + converter.afterPropertiesSet(); + MongoTemplate mongoTemplate = new MongoTemplate(mongoDbFactory(), converter); + mongoTemplate.setReadPreference(ReadPreference.valueOf(readPreference)); + mongoTemplate.setWriteConcern(WriteConcern.MAJORITY); + mongoTemplate.setWriteResultChecking(WriteResultChecking.EXCEPTION); + return mongoTemplate; + } + } diff --git a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/test/configuration/ReleaseFromDBTestConfiguration.java b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/test/configuration/ReleaseFromDBTestConfiguration.java index 4ee537858..8f84a0cf6 100644 --- a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/test/configuration/ReleaseFromDBTestConfiguration.java +++ b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/test/configuration/ReleaseFromDBTestConfiguration.java @@ -1,8 +1,9 @@ package uk.ac.ebi.eva.accession.release.test.configuration; import org.springframework.batch.core.Job; +import org.springframework.batch.core.launch.JobLauncher; +import org.springframework.batch.core.repository.JobRepository; import org.springframework.batch.test.JobLauncherTestUtils; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.context.annotation.Bean; @@ -57,46 +58,45 @@ public JobExecutionApplicationListener jobExecutionApplicationListener() { } @Bean(TEST_DUMP_ACTIVE_ACCESSIONS_JOB) - public JobLauncherTestUtils jobLauncherTestUtilsActiveAccessions() { - return new JobLauncherTestUtils() { - @Override - @Autowired - public void setJob(@Qualifier(DUMP_ACTIVE_ACCESSIONS_JOB) Job job) { - super.setJob(job); - } - }; + public JobLauncherTestUtils jobLauncherTestUtilsActiveAccessions(JobLauncher jobLauncher, JobRepository jobRepository, + @Qualifier(DUMP_ACTIVE_ACCESSIONS_JOB) Job job) { + JobLauncherTestUtils utils = new JobLauncherTestUtils(); + utils.setJobLauncher(jobLauncher); + utils.setJobRepository(jobRepository); + utils.setJob(job); + return utils; } @Bean(TEST_DUMP_MERGED_AND_DEPRECATED_ACCESSIONS_JOB) - public JobLauncherTestUtils jobLauncherTestUtilsMergedAccessions() { - return new JobLauncherTestUtils() { - @Override - @Autowired - public void setJob(@Qualifier(DUMP_MERGED_AND_DEPRECATED_ACCESSIONS_JOB) Job job) { - super.setJob(job); - } - }; + public JobLauncherTestUtils jobLauncherTestUtilsMergedAccessions(JobLauncher jobLauncher, JobRepository jobRepository, + @Qualifier(DUMP_MERGED_AND_DEPRECATED_ACCESSIONS_JOB) Job job) { + JobLauncherTestUtils utils = new JobLauncherTestUtils(); + utils.setJobLauncher(jobLauncher); + utils.setJobRepository(jobRepository); + utils.setJob(job); + return utils; + } @Bean(TEST_RELEASE_ACTIVE_ACCESSIONS_JOB) - public JobLauncherTestUtils jobLauncherTestUtilsReleaseActiveAccessions() { - return new JobLauncherTestUtils() { - @Override - @Autowired - public void setJob(@Qualifier(ACTIVE_ACCESSIONS_RELEASE_FROM_DB_JOB) Job job) { - super.setJob(job); - } - }; + public JobLauncherTestUtils jobLauncherTestUtilsReleaseActiveAccessions(JobLauncher jobLauncher, JobRepository jobRepository, + @Qualifier(ACTIVE_ACCESSIONS_RELEASE_FROM_DB_JOB) Job job) { + JobLauncherTestUtils utils = new JobLauncherTestUtils(); + utils.setJobLauncher(jobLauncher); + utils.setJobRepository(jobRepository); + utils.setJob(job); + return utils; + } @Bean(TEST_RELEASE_MERGED_AND_DEPRECATED_ACCESSIONS_JOB) - public JobLauncherTestUtils jobLauncherTestUtilsReleaseMergedAccessions() { - return new JobLauncherTestUtils() { - @Override - @Autowired - public void setJob(@Qualifier(MERGED_AND_DEPRECATED_ACCESSIONS_RELEASE_FROM_DB_JOB) Job job) { - super.setJob(job); - } - }; + public JobLauncherTestUtils jobLauncherTestUtilsReleaseMergedAccessions(JobLauncher jobLauncher, JobRepository jobRepository, + @Qualifier(MERGED_AND_DEPRECATED_ACCESSIONS_RELEASE_FROM_DB_JOB) Job job) { + JobLauncherTestUtils utils = new JobLauncherTestUtils(); + utils.setJobLauncher(jobLauncher); + utils.setJobRepository(jobRepository); + utils.setJob(job); + return utils; + } } diff --git a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/test/rule/FixSpringMongoDbRule.java b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/test/rule/FixSpringMongoDbRule.java deleted file mode 100644 index 144884962..000000000 --- a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/test/rule/FixSpringMongoDbRule.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * - * Copyright 2018 EMBL - European Bioinformatics Institute - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -package uk.ac.ebi.eva.accession.release.test.rule; - -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfiguration; -import com.lordofthejars.nosqlunit.mongodb.SpringMongoDbRule; - -/** - * Temporary fix until nosql unit rc-6 or final is released - */ -public class FixSpringMongoDbRule extends SpringMongoDbRule { - - public FixSpringMongoDbRule(MongoDbConfiguration mongoDbConfiguration) { - super(mongoDbConfiguration); - } - - public FixSpringMongoDbRule(MongoDbConfiguration mongoDbConfiguration, Object object) { - super(mongoDbConfiguration, object); - } - - @Override - public void close() { - // DO NOT CLOSE the connection (Spring will do it when destroying the context) - } - -} diff --git a/eva-accession-release/src/test/resources/application.properties b/eva-accession-release/src/test/resources/application.properties index 2f8beb508..b8fb0bd6e 100644 --- a/eva-accession-release/src/test/resources/application.properties +++ b/eva-accession-release/src/test/resources/application.properties @@ -18,9 +18,6 @@ parameters.contigNaming=SEQUENCE_NAME spring.data.mongodb.database=test-db -spring.data.mongodb.host=|eva.mongo.host.test| -spring.data.mongodb.password= -spring.data.mongodb.port=27017 mongodb.read-preference=primary # See https://github.com/spring-projects/spring-boot/wiki/Spring-Boot-2.1-Release-Notes#bean-overriding diff --git a/eva-accession-release/src/test/resources/dump-rs-accession-test.properties b/eva-accession-release/src/test/resources/dump-rs-accession-test.properties index 2cdd166f9..ee7516341 100644 --- a/eva-accession-release/src/test/resources/dump-rs-accession-test.properties +++ b/eva-accession-release/src/test/resources/dump-rs-accession-test.properties @@ -21,9 +21,6 @@ parameters.rsAccFile=src/test/resources/release-test-run/rsAccFile.csv spring.data.mongodb.database=test-db -spring.data.mongodb.host=|eva.mongo.host.test| -spring.data.mongodb.password= -spring.data.mongodb.port=27017 mongodb.read-preference=primary # See https://github.com/spring-projects/spring-boot/wiki/Spring-Boot-2.1-Release-Notes#bean-overriding diff --git a/eva-accession-release/src/test/resources/incremental-release-pipeline-test.properties b/eva-accession-release/src/test/resources/incremental-release-pipeline-test.properties index bc8a54d38..70140cc36 100644 --- a/eva-accession-release/src/test/resources/incremental-release-pipeline-test.properties +++ b/eva-accession-release/src/test/resources/incremental-release-pipeline-test.properties @@ -17,9 +17,6 @@ parameters.chunkSize=5 parameters.contigNaming=INSDC spring.data.mongodb.database=test-db -spring.data.mongodb.host=|eva.mongo.host.test| -spring.data.mongodb.password= -spring.data.mongodb.port=27017 mongodb.read-preference=primary # See https://github.com/spring-projects/spring-boot/wiki/Spring-Boot-2.1-Release-Notes#bean-overriding diff --git a/eva-accession-release/src/test/resources/release-pipeline-test.properties b/eva-accession-release/src/test/resources/release-pipeline-test.properties index de388d415..48df3dc06 100644 --- a/eva-accession-release/src/test/resources/release-pipeline-test.properties +++ b/eva-accession-release/src/test/resources/release-pipeline-test.properties @@ -18,9 +18,6 @@ parameters.contigNaming=SEQUENCE_NAME spring.data.mongodb.database=test-db -spring.data.mongodb.host=|eva.mongo.host.test| -spring.data.mongodb.password= -spring.data.mongodb.port=27017 mongodb.read-preference=primary # See https://github.com/spring-projects/spring-boot/wiki/Spring-Boot-2.1-Release-Notes#bean-overriding diff --git a/eva-accession-ws/pom.xml b/eva-accession-ws/pom.xml index d29c1428d..a9e697f1c 100644 --- a/eva-accession-ws/pom.xml +++ b/eva-accession-ws/pom.xml @@ -3,10 +3,11 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 + uk.ac.ebi.eva eva-accession - 0.6.58-SNAPSHOT + ${revision} eva-accession-ws @@ -37,20 +38,16 @@ - com.fasterxml.jackson.datatype - jackson-datatype-jsr310 - - - com.fasterxml.jackson.core - jackson-databind - - - io.springfox - springfox-swagger2 + org.springdoc + springdoc-openapi-starter-webmvc-ui + - io.springfox - springfox-swagger-ui + uk.ac.ebi.eva + eva-accession-core + ${revision} + test-jar + test @@ -59,8 +56,18 @@ - com.lordofthejars - nosqlunit-mongodb + org.springframework.boot + spring-boot-testcontainers + test + + + org.testcontainers + mongodb + test + + + org.testcontainers + junit-jupiter test @@ -98,7 +105,6 @@ - localhost ${maven.build.timestamp} yyyyMMddHHmm diff --git a/eva-accession-ws/src/main/java/uk/ac/ebi/eva/accession/ws/EvaAccessionApplication.java b/eva-accession-ws/src/main/java/uk/ac/ebi/eva/accession/ws/EvaAccessionApplication.java index becc9a0e4..a4f3e3c30 100644 --- a/eva-accession-ws/src/main/java/uk/ac/ebi/eva/accession/ws/EvaAccessionApplication.java +++ b/eva-accession-ws/src/main/java/uk/ac/ebi/eva/accession/ws/EvaAccessionApplication.java @@ -26,7 +26,7 @@ @SpringBootApplication(exclude = {MongoAutoConfiguration.class, MongoDataAutoConfiguration.class}) public class EvaAccessionApplication extends SpringBootServletInitializer { - public static void main(String[] args) throws Exception { + public static void main(String[] args) { SpringApplication.run(EvaAccessionApplication.class, args); } } diff --git a/eva-accession-ws/src/main/java/uk/ac/ebi/eva/accession/ws/configuration/ApplicationConfiguration.java b/eva-accession-ws/src/main/java/uk/ac/ebi/eva/accession/ws/configuration/ApplicationConfiguration.java index 236559315..5a25f5e8b 100644 --- a/eva-accession-ws/src/main/java/uk/ac/ebi/eva/accession/ws/configuration/ApplicationConfiguration.java +++ b/eva-accession-ws/src/main/java/uk/ac/ebi/eva/accession/ws/configuration/ApplicationConfiguration.java @@ -21,38 +21,30 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializationFeature; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; -import org.springframework.beans.factory.ObjectProvider; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.autoconfigure.AutoConfigureAfter; -import org.springframework.boot.autoconfigure.http.HttpMessageConverters; import org.springframework.boot.autoconfigure.http.HttpMessageConvertersAutoConfiguration; -import org.springframework.boot.autoconfigure.web.client.RestTemplateAutoConfiguration; +import org.springframework.boot.autoconfigure.web.client.RestTemplateBuilderConfigurer; import org.springframework.boot.autoconfigure.web.reactive.function.client.WebClientAutoConfiguration; import org.springframework.boot.web.client.RestTemplateBuilder; -import org.springframework.boot.web.client.RestTemplateCustomizer; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Import; -import org.springframework.http.client.ClientHttpRequestFactory; import org.springframework.web.servlet.config.annotation.CorsRegistry; import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; -import org.springframework.web.servlet.config.annotation.WebMvcConfigurerAdapter; import uk.ac.ebi.ampt2d.commons.accession.autoconfigure.EnableBasicRestControllerAdvice; import uk.ac.ebi.ampt2d.commons.accession.rest.controllers.BasicRestController; - +import uk.ac.ebi.eva.accession.core.configuration.human.HumanClusteredVariantAccessioningConfiguration; +import uk.ac.ebi.eva.accession.core.configuration.nonhuman.ClusteredVariantAccessioningConfiguration; +import uk.ac.ebi.eva.accession.core.configuration.nonhuman.SubmittedVariantAccessioningConfiguration; import uk.ac.ebi.eva.accession.core.model.ClusteredVariant; -import uk.ac.ebi.eva.accession.core.service.nonhuman.ClusteredVariantAccessioningService; import uk.ac.ebi.eva.accession.core.model.IClusteredVariant; import uk.ac.ebi.eva.accession.core.model.ISubmittedVariant; import uk.ac.ebi.eva.accession.core.model.SubmittedVariant; +import uk.ac.ebi.eva.accession.core.service.nonhuman.ClusteredVariantAccessioningService; import uk.ac.ebi.eva.accession.core.service.nonhuman.SubmittedVariantAccessioningService; -import uk.ac.ebi.eva.accession.core.configuration.human.HumanClusteredVariantAccessioningConfiguration; -import uk.ac.ebi.eva.accession.core.configuration.nonhuman.ClusteredVariantAccessioningConfiguration; -import uk.ac.ebi.eva.accession.core.configuration.nonhuman.SubmittedVariantAccessioningConfiguration; import uk.ac.ebi.eva.accession.ws.response.NonRedirectingClientHttpRequestFactory; -import java.util.function.Supplier; - @Configuration @EnableBasicRestControllerAdvice @Import({ClusteredVariantAccessioningConfiguration.class, SubmittedVariantAccessioningConfiguration.class, @@ -83,7 +75,7 @@ public BasicRestController ba @Bean public WebMvcConfigurer corsConfigurer() { - return new WebMvcConfigurerAdapter() { + return new WebMvcConfigurer() { @Override public void addCorsMappings(CorsRegistry registry) { registry.addMapping("/**"); @@ -97,18 +89,8 @@ public void addCorsMappings(CorsRegistry registry) { * {@link NonRedirectingClientHttpRequestFactory}. */ @Bean - public RestTemplateBuilder restTemplateBuilder( - ObjectProvider messageConverters, - ObjectProvider restTemplateCustomizers) { - - RestTemplateAutoConfiguration restTemplateConfiguration = - new RestTemplateAutoConfiguration( - messageConverters, restTemplateCustomizers); - RestTemplateBuilder builder = restTemplateConfiguration.restTemplateBuilder(); - - Supplier supplier = NonRedirectingClientHttpRequestFactory::new; - builder = builder.requestFactory(supplier); - - return builder; + public RestTemplateBuilder restTemplateBuilder(RestTemplateBuilderConfigurer configurer) { + return configurer.configure(new RestTemplateBuilder()) + .requestFactory(NonRedirectingClientHttpRequestFactory::new); } } diff --git a/eva-accession-ws/src/main/java/uk/ac/ebi/eva/accession/ws/configuration/SwaggerConfiguration.java b/eva-accession-ws/src/main/java/uk/ac/ebi/eva/accession/ws/configuration/SwaggerConfiguration.java index ee804a623..f7a89f57c 100644 --- a/eva-accession-ws/src/main/java/uk/ac/ebi/eva/accession/ws/configuration/SwaggerConfiguration.java +++ b/eva-accession-ws/src/main/java/uk/ac/ebi/eva/accession/ws/configuration/SwaggerConfiguration.java @@ -16,41 +16,41 @@ package uk.ac.ebi.eva.accession.ws.configuration; +import io.swagger.v3.oas.models.OpenAPI; +import io.swagger.v3.oas.models.info.Contact; +import io.swagger.v3.oas.models.info.Info; +import io.swagger.v3.oas.models.info.License; +import org.springdoc.core.models.GroupedOpenApi; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import springfox.documentation.builders.ApiInfoBuilder; -import springfox.documentation.builders.PathSelectors; -import springfox.documentation.builders.RequestHandlerSelectors; -import springfox.documentation.service.ApiInfo; -import springfox.documentation.service.Contact; -import springfox.documentation.spi.DocumentationType; -import springfox.documentation.spring.web.plugins.Docket; -import springfox.documentation.swagger2.annotations.EnableSwagger2; @Configuration -@EnableSwagger2 public class SwaggerConfiguration { @Bean - public Docket productApi() { - return new Docket(DocumentationType.SWAGGER_2) - .apiInfo(getApiInfo()) - .select() - .apis(RequestHandlerSelectors.basePackage("uk.ac.ebi.eva.accession.ws.rest")) - .paths(PathSelectors.any()) + public GroupedOpenApi productApi() { + return GroupedOpenApi.builder() + .group("eva-accession-api") + .packagesToScan("uk.ac.ebi.eva.accession.ws.rest") + .pathsToMatch("/**") .build(); } - private ApiInfo getApiInfo() { - return new ApiInfoBuilder() - .contact(new Contact("Europe Variation Archive - EMBL-EBI", "https://www.ebi.ac.uk/eva", null)) - .license("Apache 2.0") - .licenseUrl("http://www.apache.org/licenses/LICENSE-2.0") - .title("EVA variant identifiers API") - .description( - "API to retrieve information about submitted variant identifiers (usually called SubSNP or " + - "SS) and clustered variant identifiers (usually called RefSNP or RS).") - .version("0.1") - .build(); + @Bean + public OpenAPI customOpenAPI() { + return new OpenAPI().info( + new Info() + .title("EVA variant identifiers API") + .description( + "API to retrieve information about submitted variant identifiers " + + "(usually called SubSNP or SS) and clustered variant identifiers " + + "(usually called RefSNP or RS).") + .version("0.1") + .contact(new Contact() + .name("Europe Variation Archive - EMBL-EBI") + .url("https://www.ebi.ac.uk/eva")) + .license(new License() + .name("Apache 2.0") + .url("http://www.apache.org/licenses/LICENSE-2.0"))); } } diff --git a/eva-accession-ws/src/main/java/uk/ac/ebi/eva/accession/ws/rest/ClusteredVariantsRestController.java b/eva-accession-ws/src/main/java/uk/ac/ebi/eva/accession/ws/rest/ClusteredVariantsRestController.java index 231b6ddd6..24a2a56d1 100644 --- a/eva-accession-ws/src/main/java/uk/ac/ebi/eva/accession/ws/rest/ClusteredVariantsRestController.java +++ b/eva-accession-ws/src/main/java/uk/ac/ebi/eva/accession/ws/rest/ClusteredVariantsRestController.java @@ -17,9 +17,10 @@ */ package uk.ac.ebi.eva.accession.ws.rest; -import io.swagger.annotations.Api; -import io.swagger.annotations.ApiOperation; -import io.swagger.annotations.ApiParam; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.tags.Tag; +import jakarta.servlet.http.HttpServletResponse; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Qualifier; @@ -39,7 +40,6 @@ import uk.ac.ebi.ampt2d.commons.accession.core.models.IEvent; import uk.ac.ebi.ampt2d.commons.accession.rest.dto.AccessionResponseDTO; import uk.ac.ebi.ampt2d.commons.accession.rest.dto.HistoryEventDTO; - import uk.ac.ebi.eva.accession.core.model.ClusteredVariant; import uk.ac.ebi.eva.accession.core.model.IClusteredVariant; import uk.ac.ebi.eva.accession.core.model.ISubmittedVariant; @@ -54,7 +54,6 @@ import uk.ac.ebi.eva.commons.core.models.VariantType; import uk.ac.ebi.eva.commons.core.models.contigalias.ContigNamingConvention; -import javax.servlet.http.HttpServletResponse; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -63,7 +62,7 @@ @RestController @RequestMapping(value = "/v1/clustered-variants") -@Api(tags = {"Clustered variants"}) +@Tag(name = "Clustered variants") public class ClusteredVariantsRestController { private static final Logger logger = LoggerFactory.getLogger(ClusteredVariantsRestController.class); @@ -97,14 +96,14 @@ public ClusteredVariantsRestController( * will be thrown and a redirection to an active RS will be done by {@link EvaControllerAdvice}. Although it is * not entirely correct, it was decided to return only one of those merges as redirection, doesn't matter which one. */ - @ApiOperation(value = "Find clustered variants (RS) by identifier", notes = "This endpoint returns the clustered " + @Operation(summary = "Find clustered variants (RS) by identifier", description = "This endpoint returns the clustered " + "variants (RS) represented by the given identifier. For a description of the response, see " + "https://github.com/EBIvariation/eva-accession/wiki/Import-accessions-from-dbSNP#clustered-variant-refsnp-or-rs") @GetMapping(value = "/{identifier}", produces = "application/json") public ResponseEntity>> get( - @PathVariable @ApiParam(value = "Numerical identifier of a clustered variant, e.g.: 3000000000", + @PathVariable @Parameter(description = "Numerical identifier of a clustered variant, e.g.: 3000000000", required = true) Long identifier, - @RequestParam(required = false) @ApiParam(value = "Contig naming convention desired, default is INSDC") + @RequestParam(required = false) @Parameter(description = "Contig naming convention desired, default is INSDC") ContigNamingConvention contigNamingConvention) throws AccessionMergedException, AccessionDoesNotExistException { try { @@ -112,7 +111,7 @@ public ResponseEntity(); clusteredVariants.addAll(getNonHumanClusteredVariants(identifier, contigNamingConvention)); clusteredVariants.addAll(humanService.getAllByAccession(identifier, contigNamingConvention).stream().map(this::toDTO) - .collect(Collectors.toList())); + .collect(Collectors.toList())); if (clusteredVariants.isEmpty()) { throw new AccessionDoesNotExistException(identifier); @@ -127,32 +126,32 @@ public ResponseEntity> getVariantHistory( - @PathVariable @ApiParam(value = "Numerical identifier of a clustered variant, e.g.: 43678406", + @PathVariable @Parameter(description = "Numerical identifier of a clustered variant, e.g.: 43678406", required = true) Long identifier, - @RequestParam(required = false) @ApiParam(value = "Contig naming convention desired, default is INSDC") - ContigNamingConvention contigNamingConvention) throws AccessionDoesNotExistException { + @RequestParam(required = false) @Parameter(description = "Contig naming convention desired, default is INSDC") + ContigNamingConvention contigNamingConvention) throws AccessionDoesNotExistException { List> allVariants = new ArrayList<>(); try { allVariants.addAll(getNonHumanClusteredVariants(identifier, contigNamingConvention)); allVariants.addAll(humanService.getAllByAccession(identifier, contigNamingConvention).stream().map(this::toDTO) - .collect(Collectors.toList())); + .collect(Collectors.toList())); } catch (NoSuchElementException e) { throw new ResponseStatusException(HttpStatus.NOT_FOUND, e.getMessage()); } catch (AccessionDeprecatedException e) { allVariants.addAll(getDeprecatedClusteredVariant(identifier)); - }catch (AccessionMergedException e){ + } catch (AccessionMergedException e) { // if accession has been merged into several other accessions, we are not getting those accessions // we will rely on operations to provide info about those rs ids in which the given rs has been merged } List> allOperations = clusteredVariantOperationService.getAllOperations(identifier, contigNamingConvention) - .stream().map(this::toHistoryEventDTO).collect(Collectors.toList()); + .stream().map(this::toHistoryEventDTO).collect(Collectors.toList()); if (allVariants.isEmpty() && allOperations.isEmpty()) { throw new AccessionDoesNotExistException(identifier); @@ -173,7 +172,7 @@ private List> getSubmittedVariants( - @PathVariable @ApiParam(value = "Numerical identifier of a clustered variant, e.g.: 869808637", + @PathVariable @Parameter(description = "Numerical identifier of a clustered variant, e.g.: 869808637", required = true) Long identifier, - @RequestParam(required = false) @ApiParam(value = "Contig naming convention desired, default is INSDC") - ContigNamingConvention contigNamingConvention) + @RequestParam(required = false) @Parameter(description = "Contig naming convention desired, default is INSDC") + ContigNamingConvention contigNamingConvention) throws AccessionDoesNotExistException, AccessionDeprecatedException, AccessionMergedException { try { // trigger the checks. if the identifier was merged, the EvaControllerAdvice will redirect to the correct @@ -209,37 +208,37 @@ public List> submittedVariants = submittedVariantsService.getByClusteredVariantAccessionIn(Collections.singletonList(identifier), - contigNamingConvention); + contigNamingConvention); return submittedVariants.stream() - .map(wrapper -> new AccessionResponseDTO<>(wrapper, SubmittedVariant::new)) - .collect(Collectors.toList()); + .map(wrapper -> new AccessionResponseDTO<>(wrapper, SubmittedVariant::new)) + .collect(Collectors.toList()); } catch (NoSuchElementException e) { throw new ResponseStatusException(HttpStatus.NOT_FOUND, e.getMessage()); } } - @ApiOperation(value = "Find a clustered variant (RS) by the identifying fields", notes = "This endpoint returns " + @Operation(summary = "Find a clustered variant (RS) by the identifying fields", description = "This endpoint returns " + "the clustered variant (RS) represented by a given identifier. For a description of the response, see " + "https://github.com/EBIvariation/eva-accession/wiki/Import-accessions-from-dbSNP#clustered-variant-refsnp" + "-or-rs") @GetMapping(produces = "application/json") public ResponseEntity>> getByIdFields( - @RequestParam(name = "assemblyId") @ApiParam(value = "assembly accession in GCA format, e.g.: GCA_000002305.1") - String assembly, - @RequestParam(name = "referenceName") @ApiParam(value = "chromosome name or accession, e.g.: CM000392.2") - String chromosome, - @RequestParam(name = "start") @ApiParam(value = "start position, e.g.: 66275332") long start, + @RequestParam(name = "assemblyId") @Parameter(description = "assembly accession in GCA format, e.g.: GCA_000002305.1") + String assembly, + @RequestParam(name = "referenceName") @Parameter(description = "chromosome name or accession, e.g.: CM000392.2") + String chromosome, + @RequestParam(name = "start") @Parameter(description = "start position, e.g.: 66275332") long start, @RequestParam(name = "variantType") VariantType variantType, - @RequestParam(required = false) @ApiParam(value = "Chromosome naming convention used, default is INSDC") - ContigNamingConvention contigNamingConvention) { + @RequestParam(required = false) @Parameter(description = "Chromosome naming convention used, default is INSDC") + ContigNamingConvention contigNamingConvention) { try { List> clusteredVariants = new ArrayList<>(); List> nonHumanClusteredVariants = nonHumanActiveService.getByIdFields(assembly, chromosome, start, variantType, - contigNamingConvention); + contigNamingConvention); nonHumanClusteredVariants.stream().map(this::toDTO).forEach(clusteredVariants::add); List> humanClusteredVariants = @@ -258,24 +257,24 @@ private AccessionResponseDTO return new AccessionResponseDTO<>(clusteredVariantWrapper, ClusteredVariant::new); } - @ApiOperation(value = "Find if a clustered variant (RS) with the given identifying fields exists in our database", - notes = "This endpoint returns true or false to indicate if the RS ID is present. Optionally return the " + + @Operation(summary = "Find if a clustered variant (RS) with the given identifying fields exists in our database", + description = "This endpoint returns true or false to indicate if the RS ID is present. Optionally return the " + "RS ID.") @GetMapping(value = "/beacon/query", produces = "application/json") public BeaconAlleleResponse doesVariantExist( - @RequestParam(name = "assemblyId") @ApiParam(value = "assembly accession in GCA format, e.g.: GCA_000002305.1") - String assembly, - @RequestParam(name = "referenceName") @ApiParam(value = "chromosome name, e.g.: 16") - String chromosome, - @RequestParam(name = "start") @ApiParam(value = "start position, e.g.: 66275332") long start, + @RequestParam(name = "assemblyId") @Parameter(description = "assembly accession in GCA format, e.g.: GCA_000002305.1") + String assembly, + @RequestParam(name = "referenceName") @Parameter(description = "chromosome name, e.g.: 16") + String chromosome, + @RequestParam(name = "start") @Parameter(description = "start position, e.g.: 66275332") long start, @RequestParam(name = "variantType") VariantType variantType, @RequestParam(name = "includeDatasetReponses", required = false) - boolean includeDatasetReponses, + boolean includeDatasetReponses, HttpServletResponse response) { try { ContigNamingConvention contigNamingConvention = ContigNamingConvention.ENA_SEQUENCE_NAME; return beaconService.queryBeaconClusteredVariant(assembly, chromosome, start, variantType, - contigNamingConvention, includeDatasetReponses); + contigNamingConvention, includeDatasetReponses); } catch (Exception ex) { logger.error("Unexpected error in beacon query for chromosome={}, start={}, assembly={}, variantType={}", chromosome, start, assembly, variantType, ex); diff --git a/eva-accession-ws/src/main/java/uk/ac/ebi/eva/accession/ws/rest/EvaControllerAdvice.java b/eva-accession-ws/src/main/java/uk/ac/ebi/eva/accession/ws/rest/EvaControllerAdvice.java index e9ab9134b..d1c79691e 100644 --- a/eva-accession-ws/src/main/java/uk/ac/ebi/eva/accession/ws/rest/EvaControllerAdvice.java +++ b/eva-accession-ws/src/main/java/uk/ac/ebi/eva/accession/ws/rest/EvaControllerAdvice.java @@ -29,7 +29,7 @@ import uk.ac.ebi.ampt2d.commons.accession.rest.BasicRestControllerAdvice; import uk.ac.ebi.ampt2d.commons.accession.rest.dto.ErrorMessage; -import javax.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletRequest; import java.net.URI; /** diff --git a/eva-accession-ws/src/main/java/uk/ac/ebi/eva/accession/ws/rest/SubmittedVariantsRestController.java b/eva-accession-ws/src/main/java/uk/ac/ebi/eva/accession/ws/rest/SubmittedVariantsRestController.java index a320b474f..5aae8cf30 100644 --- a/eva-accession-ws/src/main/java/uk/ac/ebi/eva/accession/ws/rest/SubmittedVariantsRestController.java +++ b/eva-accession-ws/src/main/java/uk/ac/ebi/eva/accession/ws/rest/SubmittedVariantsRestController.java @@ -17,17 +17,18 @@ */ package uk.ac.ebi.eva.accession.ws.rest; -import io.swagger.annotations.Api; -import io.swagger.annotations.ApiOperation; -import io.swagger.annotations.ApiParam; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.tags.Tag; +import jakarta.servlet.http.HttpServletResponse; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PathVariable; -import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; import org.springframework.web.server.ResponseStatusException; import uk.ac.ebi.ampt2d.commons.accession.core.exceptions.AccessionDeprecatedException; @@ -35,7 +36,6 @@ import uk.ac.ebi.ampt2d.commons.accession.core.exceptions.AccessionMergedException; import uk.ac.ebi.ampt2d.commons.accession.core.models.AccessionWrapper; import uk.ac.ebi.ampt2d.commons.accession.rest.dto.AccessionResponseDTO; - import uk.ac.ebi.eva.accession.core.model.ISubmittedVariant; import uk.ac.ebi.eva.accession.core.model.SubmittedVariant; import uk.ac.ebi.eva.accession.core.service.nonhuman.SubmittedVariantAccessioningService; @@ -45,7 +45,6 @@ import uk.ac.ebi.eva.accession.ws.service.SubmittedVariantsBeaconService; import uk.ac.ebi.eva.commons.core.models.contigalias.ContigNamingConvention; -import javax.servlet.http.HttpServletResponse; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -54,7 +53,7 @@ @RestController @RequestMapping(value = "/v1/submitted-variants") -@Api(tags = {"Submitted variants"}) +@Tag(name = "Submitted variants") public class SubmittedVariantsRestController { private static final Logger logger = LoggerFactory.getLogger(SubmittedVariantsRestController.class); @@ -74,19 +73,19 @@ public SubmittedVariantsRestController( * will be thrown and a redirection to an active SS will be done by {@link EvaControllerAdvice}. Although it is * not entirely correct, it was decided to return only one of those merges as redirection, doesn't matter which one. */ - @ApiOperation(value = "Find submitted variants (SS) by identifier", notes = "This endpoint returns the submitted " + @Operation(summary = "Find submitted variants (SS) by identifier", description = "This endpoint returns the submitted " + "variants (SS) represented by a given identifier. For a description of the response, see " + "https://github.com/EBIvariation/eva-accession/wiki/Import-accessions-from-dbSNP#submitted-variant-subsnp-or-ss") @GetMapping(value = "/{identifier}", produces = "application/json") public ResponseEntity>> get( - @PathVariable @ApiParam(value = "Numerical identifier of a submitted variant, e.g.: 5000000000", - required = true) Long identifier, - @RequestParam(required = false) @ApiParam(value = "Contig naming convention desired, default is INSDC") + @PathVariable @Parameter(description = "Numerical identifier of a submitted variant, e.g.: 5000000000", + required = true) Long identifier, + @RequestParam(required = false) @Parameter(description = "Contig naming convention desired, default is INSDC") ContigNamingConvention contigNamingConvention) throws AccessionMergedException, AccessionDoesNotExistException { try { return ResponseEntity.ok(service.getAllByAccession(identifier, contigNamingConvention).stream() - .map(this::toDTO).collect(Collectors.toList())); + .map(this::toDTO).collect(Collectors.toList())); } catch (NoSuchElementException e) { throw new ResponseStatusException(HttpStatus.NOT_FOUND, e.getMessage()); } catch (AccessionDeprecatedException e) { @@ -103,7 +102,7 @@ private AccessionResponseDTO /** * Retrieve the information in the collection for inactive objects. - * + *

* This method is necessary because the behaviour of BasicRestController is to return the HttpStatus.GONE with an * error message in the body. We want instead to return the HttpStatus.GONE with the variant in the body. */ @@ -117,27 +116,26 @@ private List studies, - @RequestParam(name="start") @ApiParam(value = "start position, e.g.: 12856868") long start, - @RequestParam(name="referenceBases") @ApiParam(value = "reference base(s), e.g.: C") String reference, - @RequestParam(name="alternateBases") @ApiParam(value = "alternate base(s), e.g.: T") String alternate, + @RequestParam(name = "assemblyId") @Parameter(description = "assembly accession in GCA format, e.g.: GCA_000002305.1") + String assembly, + @RequestParam(name = "referenceName") @Parameter(description = "chromosome name, e.g.: 16") String chromosome, + @RequestParam(name = "datasetIds") @Parameter(description = "study accession, e.g.: PRJEB30116") List studies, + @RequestParam(name = "start") @Parameter(description = "start position, e.g.: 12856868") long start, + @RequestParam(name = "referenceBases") @Parameter(description = "reference base(s), e.g.: C") String reference, + @RequestParam(name = "alternateBases") @Parameter(description = "alternate base(s), e.g.: T") String alternate, HttpServletResponse response) { if (start < 1) { int responseStatus = HttpServletResponse.SC_BAD_REQUEST; response.setStatus(responseStatus); return getBeaconResponseObjectWithError(alternate, reference, chromosome, start, assembly, studies, - responseStatus, - "Please provide a positive number as start position"); + responseStatus, + "Please provide a positive number as start position"); } try { ContigNamingConvention contigNamingConvention = ContigNamingConvention.ENA_SEQUENCE_NAME; return submittedVariantsBeaconService.queryBeacon(studies, alternate, reference, chromosome, start, - assembly, contigNamingConvention, false); - } - catch (Exception ex) { + assembly, contigNamingConvention, false); + } catch (Exception ex) { logger.error("Unexpected error in beacon query for chromosome={}, start={}, assembly={}, studies={}", chromosome, start, assembly, studies, ex); int responseStatus = HttpServletResponse.SC_INTERNAL_SERVER_ERROR; @@ -152,32 +150,32 @@ private BeaconAlleleResponse getBeaconResponseObjectWithError(String alternate, int errorCode, String errorMessage) { BeaconAlleleResponse result = new BeaconAlleleResponse(); BeaconAlleleRequest request = new BeaconAlleleRequest(alternate, reference, chromosome, start, - assembly, studies, false); + assembly, studies, false); result.setAlleleRequest(request); result.setError(new BeaconError(errorCode, errorMessage)); return result; } - @ApiOperation(value = "Find submitted variants (SS) by the identifying fields", notes = "This endpoint returns " + @Operation(summary = "Find submitted variants (SS) by the identifying fields", description = "This endpoint returns " + "the submitted variants (SS) represented by a given identifier. For a description of the response, see " + "https://github.com/EBIvariation/eva-accession/wiki/Import-accessions-from-dbSNP#submitted-variant" + "-subsnp-or-ss") @GetMapping(produces = "application/json") public ResponseEntity>> getByIdFields( - @RequestParam(name="assemblyId") @ApiParam(value = "assembly accession in GCA format, e.g.: GCA_000002305.1") - String assembly, - @RequestParam(name="referenceName") @ApiParam(value = "chromosome name or accession, e.g.: CM000392.2") - String chromosome, - @RequestParam(name="datasetIds") @ApiParam(value = "study accession, e.g.: PRJEB30116") List studies, - @RequestParam(name="start") @ApiParam(value = "start position, e.g.: 12856868") long start, - @RequestParam(name="referenceBases") @ApiParam(value = "reference base(s), e.g.: C") String reference, - @RequestParam(name="alternateBases") @ApiParam(value = "alternate base(s), e.g.: T") String alternate, - @RequestParam(required = false) @ApiParam(value = "Chromosome naming convention used, default is INSDC") - ContigNamingConvention contigNamingConvention) { + @RequestParam(name = "assemblyId") @Parameter(description = "assembly accession in GCA format, e.g.: GCA_000002305.1") + String assembly, + @RequestParam(name = "referenceName") @Parameter(description = "chromosome name or accession, e.g.: CM000392.2") + String chromosome, + @RequestParam(name = "datasetIds") @Parameter(description = "study accession, e.g.: PRJEB30116") List studies, + @RequestParam(name = "start") @Parameter(description = "start position, e.g.: 12856868") long start, + @RequestParam(name = "referenceBases") @Parameter(description = "reference base(s), e.g.: C") String reference, + @RequestParam(name = "alternateBases") @Parameter(description = "alternate base(s), e.g.: T") String alternate, + @RequestParam(required = false) @Parameter(description = "Chromosome naming convention used, default is INSDC") + ContigNamingConvention contigNamingConvention) { try { return ResponseEntity.ok( submittedVariantsBeaconService.getVariantByIdFields(assembly, chromosome, studies, start, reference, - alternate, contigNamingConvention)); + alternate, contigNamingConvention)); } catch (NoSuchElementException e) { throw new ResponseStatusException(HttpStatus.NOT_FOUND, e.getMessage()); } catch (Exception e) { diff --git a/eva-accession-ws/src/main/java/uk/ac/ebi/eva/accession/ws/service/ClusteredVariantsBeaconService.java b/eva-accession-ws/src/main/java/uk/ac/ebi/eva/accession/ws/service/ClusteredVariantsBeaconService.java index 1a5982526..af73094b2 100644 --- a/eva-accession-ws/src/main/java/uk/ac/ebi/eva/accession/ws/service/ClusteredVariantsBeaconService.java +++ b/eva-accession-ws/src/main/java/uk/ac/ebi/eva/accession/ws/service/ClusteredVariantsBeaconService.java @@ -17,7 +17,6 @@ */ package uk.ac.ebi.eva.accession.ws.service; -import org.apache.commons.collections.CollectionUtils; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Service; import uk.ac.ebi.ampt2d.commons.accession.core.models.AccessionWrapper; @@ -167,7 +166,7 @@ private BeaconAlleleResponse queryBeaconClusteredVariantHuman(String referenceGe getBeaconDatasetAlleleResponsesHuman = getBeaconDatasetAlleleResponsesHuman(identifier); } return buildResponse(referenceGenome, chromosome, start, variantType, !variant.isEmpty(), - getBeaconDatasetAlleleResponsesHuman); + getBeaconDatasetAlleleResponsesHuman); } private List getBeaconDatasetAlleleResponsesHuman(String clusteredVariantAccession) { @@ -193,13 +192,13 @@ private BeaconAlleleResponse mergeResponses(BeaconAlleleResponse nonHumanRespons if (response.isExists()) { List datasetAlleleResponses = new ArrayList<>(); - if (!CollectionUtils.isEmpty(nonHumanResponse.getDatasetAlleleResponses())) { + if (nonHumanResponse.getDatasetAlleleResponses() != null && !nonHumanResponse.getDatasetAlleleResponses().isEmpty()) { datasetAlleleResponses.addAll(nonHumanResponse.getDatasetAlleleResponses()); } - if (!CollectionUtils.isEmpty(humanResponse.getDatasetAlleleResponses())) { + if (humanResponse.getDatasetAlleleResponses() != null && !humanResponse.getDatasetAlleleResponses().isEmpty()) { datasetAlleleResponses.addAll(humanResponse.getDatasetAlleleResponses()); } - if (!CollectionUtils.isEmpty(datasetAlleleResponses)) { + if (datasetAlleleResponses != null && !datasetAlleleResponses.isEmpty()) { response.setDatasetAlleleResponses(datasetAlleleResponses); } } @@ -208,8 +207,8 @@ private BeaconAlleleResponse mergeResponses(BeaconAlleleResponse nonHumanRespons } public BeaconAlleleResponse getBeaconResponseObjectWithError(String reference, long start, String assembly, - VariantType variantType, int errorCode, - String errorMessage) { + VariantType variantType, int errorCode, + String errorMessage) { BeaconAlleleRequest request = new BeaconAlleleRequest(); request.setReferenceName(Chromosome.fromValue(reference)); request.setStart(start); diff --git a/eva-accession-ws/src/test/java/uk/ac/ebi/eva/accession/ws/ClusteredVariantHistoryEndPointTest.java b/eva-accession-ws/src/test/java/uk/ac/ebi/eva/accession/ws/ClusteredVariantHistoryEndPointTest.java index e56979750..9a716ab72 100644 --- a/eva-accession-ws/src/test/java/uk/ac/ebi/eva/accession/ws/ClusteredVariantHistoryEndPointTest.java +++ b/eva-accession-ws/src/test/java/uk/ac/ebi/eva/accession/ws/ClusteredVariantHistoryEndPointTest.java @@ -17,81 +17,80 @@ */ package uk.ac.ebi.eva.accession.ws; -import com.lordofthejars.nosqlunit.annotation.UsingDataSet; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; -import com.mongodb.MongoClient; -import org.junit.After; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.web.client.TestRestTemplate; -import org.springframework.context.ApplicationContext; +import org.springframework.context.annotation.Import; import org.springframework.core.ParameterizedTypeReference; +import org.springframework.core.io.ResourceLoader; +import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.http.HttpMethod; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.ampt2d.commons.accession.core.models.EventType; import uk.ac.ebi.ampt2d.commons.accession.rest.dto.AccessionResponseDTO; import uk.ac.ebi.ampt2d.commons.accession.rest.dto.HistoryEventDTO; - import uk.ac.ebi.eva.accession.core.model.ClusteredVariant; import uk.ac.ebi.eva.accession.core.model.IClusteredVariant; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; +import uk.ac.ebi.eva.accession.core.utils.MongoTestDataLoader; import uk.ac.ebi.eva.accession.ws.dto.VariantHistory; import uk.ac.ebi.eva.accession.ws.rest.ClusteredVariantsRestController; -import uk.ac.ebi.eva.accession.ws.test.rule.FixSpringMongoDbRule; +import uk.ac.ebi.eva.accession.ws.test.MongoTestConfiguration; import uk.ac.ebi.eva.commons.core.models.contigalias.ContigNamingConvention; import java.util.Arrays; import java.util.List; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; -@RunWith(SpringRunner.class) +@ExtendWith(SpringExtension.class) @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +@Import({MongoTestConfiguration.class}) @TestPropertySource("classpath:accession-ws-test.properties") -public class ClusteredVariantHistoryEndPointTest { +public class ClusteredVariantHistoryEndPointTest extends MongoTestContainerHelper { private static final String URL = "/v1/clustered-variants/%s/history"; - public static final String TEST_DB = "eva-accession-ws-test-db"; - @Autowired private TestRestTemplate testRestTemplate; @Autowired - private MongoClient mongoClient; + private MongoTemplate mongoTemplate; @Autowired - private ClusteredVariantsRestController restController; + private ResourceLoader resourceLoader; - //Required by nosql-unit @Autowired - private ApplicationContext applicationContext; + private ClusteredVariantsRestController restController; - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName(TEST_DB).build()); + @BeforeEach + public void setUp() { + mongoTemplate.getDb().drop(); + } - @After + @AfterEach public void tearDown() { - mongoClient.dropDatabase(TEST_DB); + mongoTemplate.getDb().drop(); } private static class ClusteredVariantType extends ParameterizedTypeReference< VariantHistory> { } - @UsingDataSet(locations = {"/test-data/splitOneRsIntoMultiple.json"}) @Test @DirtiesContext public void testVariantHistorySingleRSSplitIntoMultiple1() throws Exception { + new MongoTestDataLoader(mongoTemplate, resourceLoader).loadAll("/test-data/splitOneRsIntoMultiple.json"); + long fetchHistoryOfRS = 1L; ResponseEntity> response = restController.getVariantHistory(fetchHistoryOfRS, ContigNamingConvention.INSDC); @@ -105,18 +104,19 @@ public void testVariantHistorySingleRSSplitIntoMultiple1() throws Exception { assertTrue(allVariants.stream().allMatch(v -> v.getAccession().equals(1L))); //Variant with RS 1 present in both ASM1 and ASM2 assertTrue(allVariants.stream().allMatch(v -> Arrays.asList("ASM1", "ASM2") - .contains(v.getData().getAssemblyAccession()))); + .contains(v.getData().getAssemblyAccession()))); assertTrue(allOperations.stream().allMatch(o -> o.getType().equals(EventType.RS_SPLIT))); // Every split operation from RS 1 assertTrue(allOperations.stream().allMatch(o -> o.getAccession().equals(1L))); assertTrue(allOperations.stream().allMatch(o -> Arrays.asList(3000000000L, 3000000001L, 3000000002L) - .contains(o.getSplitInto()))); + .contains(o.getSplitInto()))); } - @UsingDataSet(locations = {"/test-data/rsMerge.Json"}) @Test @DirtiesContext public void testVariantHistoryRSMerge() throws Exception { + new MongoTestDataLoader(mongoTemplate, resourceLoader).loadAll("/test-data/rsMerge.Json"); + long fetchHistoryOfRS = 1L; ResponseEntity> response = restController.getVariantHistory(fetchHistoryOfRS, ContigNamingConvention.INSDC); @@ -130,17 +130,18 @@ public void testVariantHistoryRSMerge() throws Exception { assertTrue(allVariants.stream().allMatch(v -> v.getAccession().equals(1L))); //Variant present in both ASM1 and ASM2 assertTrue(allVariants.stream().allMatch(v -> Arrays.asList("ASM1", "ASM2") - .contains(v.getData().getAssemblyAccession()))); + .contains(v.getData().getAssemblyAccession()))); assertTrue(allOperations.stream().allMatch(o -> o.getType().equals(EventType.MERGED))); // Operation RS 2 merged into 1 assertEquals(2L, allOperations.get(0).getAccession().longValue()); assertEquals(1L, allOperations.get(0).getMergedInto().longValue()); } - @UsingDataSet(locations = {"/test-data/subsequentRsSplit.json"}) @Test @DirtiesContext public void testVariantHistorySubsequentRSSplit() throws Exception { + new MongoTestDataLoader(mongoTemplate, resourceLoader).loadAll("/test-data/subsequentRsSplit.json"); + long fetchHistoryOfRS = 3000000000L; ResponseEntity> response = restController.getVariantHistory(fetchHistoryOfRS, ContigNamingConvention.INSDC); @@ -154,17 +155,18 @@ public void testVariantHistorySubsequentRSSplit() throws Exception { assertTrue(allVariants.stream().allMatch(v -> v.getAccession().equals(3000000000L))); //Variant present in both ASM2 and ASM3 assertTrue(allVariants.stream().allMatch(v -> Arrays.asList("ASM2", "ASM3") - .contains(v.getData().getAssemblyAccession()))); + .contains(v.getData().getAssemblyAccession()))); assertTrue(allOperations.stream().allMatch(o -> o.getType().equals(EventType.RS_SPLIT))); assertTrue(allOperations.stream().allMatch(o -> Arrays.asList(1L, 3000000000L).contains(o.getAccession()))); assertTrue(allOperations.stream().allMatch(o -> Arrays.asList(3000000000L, 3000000001L) - .contains(o.getSplitInto()))); + .contains(o.getSplitInto()))); } - @UsingDataSet(locations = {"/test-data/subsequentRsMerge.json"}) @Test @DirtiesContext public void testVariantHistorySubsequentRSMerge() throws Exception { + new MongoTestDataLoader(mongoTemplate, resourceLoader).loadAll("/test-data/subsequentRsMerge.json"); + long fetchHistoryOfRS = 9L; ResponseEntity> response = restController.getVariantHistory(fetchHistoryOfRS, ContigNamingConvention.INSDC); @@ -183,10 +185,11 @@ public void testVariantHistorySubsequentRSMerge() throws Exception { assertTrue(allOperations.stream().allMatch(o -> Arrays.asList(9L, 8L).contains(o.getMergedInto()))); } - @UsingDataSet(locations = {"/test-data/mixOfSplitAndMerge.json"}) @Test @DirtiesContext public void testVariantHistoryMixOfSplitAndMerge() throws Exception { + new MongoTestDataLoader(mongoTemplate, resourceLoader).loadAll("/test-data/mixOfSplitAndMerge.json"); + long fetchHistoryOfRS = 3000000000L; ResponseEntity> response = restController.getVariantHistory(fetchHistoryOfRS, ContigNamingConvention.INSDC); @@ -209,10 +212,11 @@ public void testVariantHistoryMixOfSplitAndMerge() throws Exception { assertEquals(8L, allOperations.get(1).getMergedInto().longValue()); } - @UsingDataSet(locations = {"/test-data/rsSplitsEndpoint.json"}) @Test @DirtiesContext public void testVariantHistoryServiceEndPoint() { + new MongoTestDataLoader(mongoTemplate, resourceLoader).loadAll("/test-data/rsSplitsEndpoint.json"); + int fetchHistoryOfRS = 1; String getVariantsUrl = String.format(URL, fetchHistoryOfRS); @@ -231,11 +235,11 @@ public void testVariantHistoryServiceEndPoint() { assertTrue(allVariants.stream().allMatch(v -> v.getAccession().equals(1L))); //Variant with RS 1 present in both ASM1 and ASM2 assertTrue(allVariants.stream().allMatch(v -> Arrays.asList("ASM1", "ASM2") - .contains(v.getData().getAssemblyAccession()))); + .contains(v.getData().getAssemblyAccession()))); assertTrue(allOperations.stream().allMatch(o -> o.getType().equals(EventType.RS_SPLIT))); // Every split operation from RS 1 assertTrue(allOperations.stream().allMatch(o -> o.getAccession().equals(1L))); assertTrue(allOperations.stream().allMatch(o -> Arrays.asList(3000000000L, 3000000001L, 3000000002L) - .contains(o.getSplitInto()))); + .contains(o.getSplitInto()))); } } \ No newline at end of file diff --git a/eva-accession-ws/src/test/java/uk/ac/ebi/eva/accession/ws/ClusteredVariantsRestControllerTest.java b/eva-accession-ws/src/test/java/uk/ac/ebi/eva/accession/ws/ClusteredVariantsRestControllerTest.java index c3e6af8ca..3718a92b6 100644 --- a/eva-accession-ws/src/test/java/uk/ac/ebi/eva/accession/ws/ClusteredVariantsRestControllerTest.java +++ b/eva-accession-ws/src/test/java/uk/ac/ebi/eva/accession/ws/ClusteredVariantsRestControllerTest.java @@ -18,11 +18,12 @@ package uk.ac.ebi.eva.accession.ws; import com.mongodb.BasicDBObject; +import jakarta.servlet.http.HttpServletResponse; import org.bson.Document; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.Mockito; import org.springframework.beans.factory.annotation.Autowired; @@ -39,7 +40,7 @@ import org.springframework.http.ResponseEntity; import org.springframework.mock.web.MockHttpServletResponse; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import org.springframework.web.server.ResponseStatusException; import uk.ac.ebi.ampt2d.commons.accession.core.exceptions.AccessionDeprecatedException; import uk.ac.ebi.ampt2d.commons.accession.core.exceptions.AccessionDoesNotExistException; @@ -71,8 +72,10 @@ import uk.ac.ebi.eva.accession.core.service.nonhuman.dbsnp.DbsnpClusteredVariantMonotonicAccessioningService; import uk.ac.ebi.eva.accession.core.summary.ClusteredVariantSummaryFunction; import uk.ac.ebi.eva.accession.core.summary.SubmittedVariantSummaryFunction; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.ws.rest.ClusteredVariantsRestController; import uk.ac.ebi.eva.accession.ws.service.ClusteredVariantsBeaconService; +import uk.ac.ebi.eva.accession.ws.test.MongoTestConfiguration; import uk.ac.ebi.eva.accession.ws.test.NoContigTranslationArgumentMatcher; import uk.ac.ebi.eva.commons.beacon.models.BeaconAlleleRequest; import uk.ac.ebi.eva.commons.beacon.models.BeaconAlleleResponse; @@ -81,7 +84,6 @@ import uk.ac.ebi.eva.commons.core.models.VariantType; import uk.ac.ebi.eva.commons.core.models.contigalias.ContigNamingConvention; -import javax.servlet.http.HttpServletResponse; import java.time.LocalDateTime; import java.time.Month; import java.util.ArrayList; @@ -96,22 +98,23 @@ import java.util.stream.Stream; import static junit.framework.TestCase.assertTrue; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertThrows; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.argThat; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.when; -@RunWith(SpringRunner.class) +@ExtendWith(SpringExtension.class) @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) -@Import({ClusteredVariantAccessioningConfiguration.class, SubmittedVariantAccessioningConfiguration.class}) +@Import({ClusteredVariantAccessioningConfiguration.class, SubmittedVariantAccessioningConfiguration.class, + MongoTestConfiguration.class}) @TestPropertySource("classpath:accession-ws-test.properties") -public class ClusteredVariantsRestControllerTest { +public class ClusteredVariantsRestControllerTest extends MongoTestContainerHelper { private static final String URL = "/v1/clustered-variants/"; @@ -217,7 +220,7 @@ public class ClusteredVariantsRestControllerTest { @MockBean private ContigAliasService contigAliasService; - @Before + @BeforeEach public void setUp() { dbsnpRepository.deleteAll(); dbsnpSubmittedVariantRepository.deleteAll(); @@ -235,9 +238,9 @@ public void setUp() { ClusteredVariantsBeaconService mockBeaconService = Mockito.spy( new ClusteredVariantsBeaconService(clusteredService, mockHumanService, mockService)); Mockito.doThrow(new RuntimeException("Some unexpected error")).when(mockBeaconService) - .queryBeaconClusteredVariant("GCA_ERROR", "CHROM1", 123, VariantType.SNV, ContigNamingConvention.ENA_SEQUENCE_NAME, false); + .queryBeaconClusteredVariant("GCA_ERROR", "CHROM1", 123, VariantType.SNV, ContigNamingConvention.ENA_SEQUENCE_NAME, false); Mockito.doThrow(new RuntimeException("Some unexpected error")).when(mockHumanService) - .getByIdFields("GCA_ERROR", "CHROM1", 123, VariantType.SNV, ContigNamingConvention.INSDC); + .getByIdFields("GCA_ERROR", "CHROM1", 123, VariantType.SNV, ContigNamingConvention.INSDC); mockController = new ClusteredVariantsRestController(mockService, mockBeaconService, mockHumanService, clusteredService, clusteredVariantOperationService ); @@ -245,26 +248,26 @@ public void setUp() { private void setupDbSnpClusteredVariants() { ClusteredVariant variant1 = new ClusteredVariant("ASMACC01", 1101, "CHROM1", 1234, VariantType.SNV, false, - null); + null); ClusteredVariant variant2 = new ClusteredVariant("ASMACC01", 1102, "CHROM1", 1234, VariantType.MNV, true, null); ClusteredVariant variant3 = new ClusteredVariant("ASMACC01", 1102, "CHROM1", 4567, VariantType.SNV, false, - null); + null); Function function = new ClusteredVariantSummaryFunction().andThen(new SHA1HashingFunction()); clusteredVariantEntity1 = new DbsnpClusteredVariantEntity(DBSNP_CLUSTERED_VARIANT_ACCESSION_1, - function.apply(variant1), variant1); + function.apply(variant1), variant1); clusteredVariantEntity2 = new DbsnpClusteredVariantEntity(DBSNP_CLUSTERED_VARIANT_ACCESSION_2, - function.apply(variant2), variant2); + function.apply(variant2), variant2); clusteredVariantEntity3 = new DbsnpClusteredVariantEntity(DBSNP_CLUSTERED_VARIANT_ACCESSION_3, - function.apply(variant3), variant3); + function.apply(variant3), variant3); // No new dbSNP accessions can be generated, so the variants can only be stored directly using a repository // TODO When the support for new EVA accessions is implemented, this could be changed // In order to do so, replicate the structure of {@link SubmittedVariantsRestControllerTest} generatedAccessions = dbsnpRepository.saveAll(Arrays.asList(clusteredVariantEntity1, clusteredVariantEntity2, - clusteredVariantEntity3)); + clusteredVariantEntity3)); setUpContigAliasMock(); } @@ -311,12 +314,12 @@ private void setUpContigAliasMock() { private void setupDbSnpClusteredHumanVariants() { ClusteredVariant variant1 = new ClusteredVariant("GCA_000001405.27", 9606, "CM000684.2", 45565333L, - VariantType.SNV, false, - LocalDateTime.of(2000, Month.SEPTEMBER, 19, 18, 2, 0)); + VariantType.SNV, false, + LocalDateTime.of(2000, Month.SEPTEMBER, 19, 18, 2, 0)); ClusteredVariant variant2 = new ClusteredVariant("GCA_000001405.27", 9606, "CM000663.2", 1234L, - VariantType.SNV, false, - LocalDateTime.of(2000, Month.SEPTEMBER, 19, 18, 2, 0)); + VariantType.SNV, false, + LocalDateTime.of(2000, Month.SEPTEMBER, 19, 18, 2, 0)); Function function = new ClusteredVariantSummaryFunction().andThen(new SHA1HashingFunction()); @@ -353,23 +356,23 @@ private void setupDbSnpClusteredHumanOperations() { private void setupDbsnpSubmittedVariants() { // one variant has default flags, the other have no default values SubmittedVariant submittedVariant1 = new SubmittedVariant("ASMACC01", 1101, "PROJECT1", "CHROM1", 1234, "REF", - "ALT", DBSNP_CLUSTERED_VARIANT_ACCESSION_1); + "ALT", DBSNP_CLUSTERED_VARIANT_ACCESSION_1); SubmittedVariant submittedVariant2 = new SubmittedVariant("ASMACC01", 1102, "PROJECT1", "CHROM1", 2345, "REF", - "ALT", DBSNP_CLUSTERED_VARIANT_ACCESSION_2, - !ISubmittedVariant.DEFAULT_SUPPORTED_BY_EVIDENCE, - !ISubmittedVariant.DEFAULT_ASSEMBLY_MATCH, - !ISubmittedVariant.DEFAULT_ALLELES_MATCH, - !ISubmittedVariant.DEFAULT_VALIDATED, null); + "ALT", DBSNP_CLUSTERED_VARIANT_ACCESSION_2, + !ISubmittedVariant.DEFAULT_SUPPORTED_BY_EVIDENCE, + !ISubmittedVariant.DEFAULT_ASSEMBLY_MATCH, + !ISubmittedVariant.DEFAULT_ALLELES_MATCH, + !ISubmittedVariant.DEFAULT_VALIDATED, null); SubmittedVariantSummaryFunction submittedVariantSummaryFunction = new SubmittedVariantSummaryFunction(); submittedVariantEntity1 = new DbsnpSubmittedVariantEntity(DBSNP_SUBMITTED_VARIANT_ACCESSION_1, - submittedVariantSummaryFunction.apply(submittedVariant1), - submittedVariant1, VERSION_1); + submittedVariantSummaryFunction.apply(submittedVariant1), + submittedVariant1, VERSION_1); submittedVariantEntity2 = new DbsnpSubmittedVariantEntity(DBSNP_SUBMITTED_VARIANT_ACCESSION_2, - submittedVariantSummaryFunction.apply(submittedVariant2), - submittedVariant2, VERSION_1); + submittedVariantSummaryFunction.apply(submittedVariant2), + submittedVariant2, VERSION_1); dbsnpSubmittedVariantRepository.saveAll(Arrays.asList(submittedVariantEntity1, submittedVariantEntity2)); } @@ -377,28 +380,28 @@ private void setupDbsnpSubmittedVariants() { private void setupEvaSubmittedVariants() { // one variant has no default flags, while the others have the default values SubmittedVariant submittedVariant3 = new SubmittedVariant("ASMACC01", 1102, "EVAPROJECT1", "CHROM1", 1234, - "REF", "ALT", DBSNP_CLUSTERED_VARIANT_ACCESSION_2); + "REF", "ALT", DBSNP_CLUSTERED_VARIANT_ACCESSION_2); SubmittedVariant submittedVariant4 = new SubmittedVariant("ASMACC01", 1102, "EVAPROJECT1", "CHROM1", 4567, - "REF", "ALT", DBSNP_CLUSTERED_VARIANT_ACCESSION_3, - !ISubmittedVariant.DEFAULT_SUPPORTED_BY_EVIDENCE, - !ISubmittedVariant.DEFAULT_ASSEMBLY_MATCH, - !ISubmittedVariant.DEFAULT_ALLELES_MATCH, - !ISubmittedVariant.DEFAULT_VALIDATED, null); + "REF", "ALT", DBSNP_CLUSTERED_VARIANT_ACCESSION_3, + !ISubmittedVariant.DEFAULT_SUPPORTED_BY_EVIDENCE, + !ISubmittedVariant.DEFAULT_ASSEMBLY_MATCH, + !ISubmittedVariant.DEFAULT_ALLELES_MATCH, + !ISubmittedVariant.DEFAULT_VALIDATED, null); SubmittedVariantSummaryFunction submittedVariantSummaryFunction = new SubmittedVariantSummaryFunction(); evaSubmittedVariantEntity3 = new SubmittedVariantEntity(EVA_SUBMITTED_VARIANT_ACCESSION_1, - submittedVariantSummaryFunction.apply(submittedVariant3), - submittedVariant3, VERSION_1); + submittedVariantSummaryFunction.apply(submittedVariant3), + submittedVariant3, VERSION_1); evaSubmittedVariantEntity4 = new SubmittedVariantEntity(EVA_SUBMITTED_VARIANT_ACCESSION_2, - submittedVariantSummaryFunction.apply(submittedVariant4), - submittedVariant4, VERSION_2); + submittedVariantSummaryFunction.apply(submittedVariant4), + submittedVariant4, VERSION_2); submittedVariantRepository.saveAll(Arrays.asList(evaSubmittedVariantEntity3, evaSubmittedVariantEntity4)); } - @After + @AfterEach public void tearDown() { dbsnpRepository.deleteAll(); dbsnpSubmittedVariantRepository.deleteAll(); @@ -412,10 +415,10 @@ public void tearDown() { @Test public void checkIndexInactiveObjectHashedMessageOnlyInHumanDB() { assertFalse(isIndexInCollection(mongoTemplate, DBSNP_CLUSTERED_VARIANT_OPERATION_ENTITY, - INACTIVE_OBJECTS_HASHED_MESSAGE)); + INACTIVE_OBJECTS_HASHED_MESSAGE)); assertTrue(isIndexInCollection(humanMongoTemplate, DBSNP_CLUSTERED_VARIANT_OPERATION_ENTITY, - INACTIVE_OBJECTS_HASHED_MESSAGE)); + INACTIVE_OBJECTS_HASHED_MESSAGE)); } private boolean isIndexInCollection(MongoTemplate template, String collection, String indexName) { @@ -493,11 +496,11 @@ private void checkClusteredVariantsOutput( Long accession) { List> expectedVariants = Stream.of(clusteredVariantEntity1, clusteredVariantEntity2, clusteredVariantEntity3) - .filter(v -> v.getAccession().equals(accession)) - .collect(Collectors.toList()); + .filter(v -> v.getAccession().equals(accession)) + .collect(Collectors.toList()); assertVariantsAreContainedInControllerResponse(getVariantsResponse, - expectedVariants, - ClusteredVariant::new); + expectedVariants, + ClusteredVariant::new); assertClusteredVariantCreatedDateNotNull(getVariantsResponse); } @@ -515,22 +518,22 @@ private void assertVariantsAreContainedInControllerResponse( Function modelToDto) { // check the accessions returned by the service Set retrievedAccessions = getVariantsResponse.stream() - .map(AccessionResponseDTO::getAccession) - .collect(Collectors.toSet()); + .map(AccessionResponseDTO::getAccession) + .collect(Collectors.toSet()); assertTrue(expectedVariants.stream() - .map(AccessionedDocument::getAccession) - .allMatch(retrievedAccessions::contains)); + .map(AccessionedDocument::getAccession) + .allMatch(retrievedAccessions::contains)); // check the objects returned by the service Set variantsReturnedByController = getVariantsResponse.stream() - .map(AccessionResponseDTO::getData) - .collect(Collectors.toSet()); + .map(AccessionResponseDTO::getData) + .collect(Collectors.toSet()); assertTrue(expectedVariants.stream() - .map(AccessionedDocument::getModel) - .map(modelToDto) - .allMatch(variantsReturnedByController::contains)); + .map(AccessionedDocument::getModel) + .map(modelToDto) + .allMatch(variantsReturnedByController::contains)); } private void assertClusteredVariantCreatedDateNotNull( @@ -565,13 +568,13 @@ private void checkSubmittedVariantsOutput( Long accession) { List> expectedVariants = Stream.of(submittedVariantEntity1, submittedVariantEntity2, evaSubmittedVariantEntity3, - evaSubmittedVariantEntity4) - .filter(v -> v.getAccession().equals(accession)) - .collect(Collectors.toList()); + evaSubmittedVariantEntity4) + .filter(v -> v.getAccession().equals(accession)) + .collect(Collectors.toList()); assertVariantsAreContainedInControllerResponse(getSubmittedVariantsReponse, - expectedVariants, - SubmittedVariant::new); + expectedVariants, + SubmittedVariant::new); assertSubmittedVariantCreatedDateNotNull(getSubmittedVariantsReponse); } @@ -619,12 +622,12 @@ public void testGetSubmittedVariantsByClusteredVariantIds_withContigTranslation( throws AccessionMergedException, AccessionDoesNotExistException, AccessionDeprecatedException { List> getVariantsResponse = controller.getSubmittedVariants(DBSNP_CLUSTERED_VARIANT_ACCESSION_1, - ContigNamingConvention.ENA_SEQUENCE_NAME); + ContigNamingConvention.ENA_SEQUENCE_NAME); DbsnpSubmittedVariantEntity expectedSubmittedVariant = getDbsnpSubmittedVariantEntityWithEnaContigName( submittedVariantEntity1); assertVariantsAreContainedInControllerResponse(getVariantsResponse, - Collections.singletonList(expectedSubmittedVariant), - SubmittedVariant::new); + Collections.singletonList(expectedSubmittedVariant), + SubmittedVariant::new); } private DbsnpSubmittedVariantEntity getDbsnpSubmittedVariantEntityWithEnaContigName( @@ -655,8 +658,8 @@ private void getAndCheckSubmittedVariantsByClusteredVariantIds(Long clusteredVar List> getVariantsResponse = controller.getSubmittedVariants(clusteredVariantIds, ContigNamingConvention.INSDC); assertVariantsAreContainedInControllerResponse(getVariantsResponse, - expectedSubmittedVariants, - SubmittedVariant::new); + expectedSubmittedVariants, + SubmittedVariant::new); } @Test @@ -664,13 +667,13 @@ public void testGetRedirectionForMergedVariants() throws AccessionMergedException, AccessionDoesNotExistException, AccessionDeprecatedException { // given clusteredService.merge(DBSNP_CLUSTERED_VARIANT_ACCESSION_1, - DBSNP_CLUSTERED_VARIANT_ACCESSION_2, - "Just for testing the endpoint, let's pretend the variants are equivalent"); + DBSNP_CLUSTERED_VARIANT_ACCESSION_2, + "Just for testing the endpoint, let's pretend the variants are equivalent"); // when String getVariantsUrl = URL + DBSNP_CLUSTERED_VARIANT_ACCESSION_1; ResponseEntity firstResponse = testRestTemplate.exchange(getVariantsUrl, HttpMethod.GET, null, - String.class); + String.class); // then assertEquals(HttpStatus.MOVED_PERMANENTLY, firstResponse.getStatusCode()); @@ -686,7 +689,7 @@ public void testGetRedirectionForMergedVariants() assertEquals(HttpStatus.OK, getVariantsResponse.getStatusCode()); assertEquals(1, getVariantsResponse.getBody().size()); assertEquals(new Long(DBSNP_CLUSTERED_VARIANT_ACCESSION_2), - getVariantsResponse.getBody().get(0).getAccession()); + getVariantsResponse.getBody().get(0).getAccession()); assertClusteredVariantCreatedDateNotNull(getVariantsResponse.getBody()); } @@ -695,13 +698,13 @@ public void testGetRedirectionForSubmittedVariantByMergedClusteredVariant() throws AccessionMergedException, AccessionDoesNotExistException, AccessionDeprecatedException { // given clusteredService.merge(DBSNP_CLUSTERED_VARIANT_ACCESSION_1, - DBSNP_CLUSTERED_VARIANT_ACCESSION_2, - "Just for testing the endpoint, let's pretend the variants are equivalent"); + DBSNP_CLUSTERED_VARIANT_ACCESSION_2, + "Just for testing the endpoint, let's pretend the variants are equivalent"); // when String getVariantsUrl = URL + DBSNP_CLUSTERED_VARIANT_ACCESSION_1 + "/submitted"; ResponseEntity firstResponse = testRestTemplate.exchange(getVariantsUrl, HttpMethod.GET, null, - String.class); + String.class); // then assertEquals(HttpStatus.MOVED_PERMANENTLY, firstResponse.getStatusCode()); @@ -719,7 +722,7 @@ public void testGetRedirectionForSubmittedVariantByMergedClusteredVariant() for (AccessionResponseDTO bodyEntry : getVariantsResponse.getBody()) { assertEquals(new Long(DBSNP_CLUSTERED_VARIANT_ACCESSION_2), - bodyEntry.getData().getClusteredVariantAccession()); + bodyEntry.getData().getClusteredVariantAccession()); } assertSubmittedVariantCreatedDateNotNull(getVariantsResponse.getBody()); @@ -735,46 +738,46 @@ public void testGetVariantsMergedSeveralTimes() // given Long outdatedAccession = 1L; ClusteredVariant variant1 = new ClusteredVariant("ASMACC01", 2000, "CHROM1", 1234, VariantType.SNV, false, - null); + null); DbsnpClusteredVariantEntity clusteredVariantEntity1 = new DbsnpClusteredVariantEntity(outdatedAccession, - "hash-100", variant1, 1); + "hash-100", variant1, 1); ClusteredVariant variant2 = new ClusteredVariant("ASMACC02", 2000, "CHROM2", 1234, VariantType.SNV, false, - null); + null); DbsnpClusteredVariantEntity clusteredVariantEntity2 = new DbsnpClusteredVariantEntity(outdatedAccession, - "hash-200", variant2, 1); + "hash-200", variant2, 1); Long currentAccession = 2L; ClusteredVariant variant4 = new ClusteredVariant("ASMACC03", 2000, "CHROM2", 1234, VariantType.SNV, false, - null); + null); DbsnpClusteredVariantEntity clusteredVariantEntity4 = new DbsnpClusteredVariantEntity(currentAccession, - "hash-400", variant4, 1); + "hash-400", variant4, 1); Long anotherCurrentAccession = 3L; ClusteredVariant variant5 = new ClusteredVariant("ASMACC04", 2000, "CHROM2", 1234, VariantType.SNV, false, - null); + null); DbsnpClusteredVariantEntity clusteredVariantEntity5 = new DbsnpClusteredVariantEntity(anotherCurrentAccession, - "hash-500", variant5, 1); + "hash-500", variant5, 1); mongoTemplate.dropCollection(DbsnpClusteredVariantEntity.class); mongoTemplate.insert(Arrays.asList(clusteredVariantEntity1, clusteredVariantEntity4, clusteredVariantEntity5), - DbsnpClusteredVariantEntity.class); + DbsnpClusteredVariantEntity.class); dbsnpService.merge(outdatedAccession, currentAccession, - "Just for testing the endpoint, let's pretend the variants are equivalent"); + "Just for testing the endpoint, let's pretend the variants are equivalent"); mongoTemplate.insert(Arrays.asList(clusteredVariantEntity2), DbsnpClusteredVariantEntity.class); dbsnpService.merge(outdatedAccession, anotherCurrentAccession, - "Second merge. This can totally happen importing from dbSNP. See rs106458077"); + "Second merge. This can totally happen importing from dbSNP. See rs106458077"); // when String getVariantsUrl = URL + outdatedAccession; ResponseEntity firstResponse = testRestTemplate.exchange(getVariantsUrl, HttpMethod.GET, null, - String.class); + String.class); // then assertEquals(HttpStatus.MOVED_PERMANENTLY, firstResponse.getStatusCode()); String redirectUrlIncludingHostAndPort = firstResponse.getHeaders().get(HttpHeaders.LOCATION).get(0); String redirectedUrl = redirectUrlIncludingHostAndPort.substring(redirectUrlIncludingHostAndPort.indexOf(URL)); assertTrue((URL + currentAccession).equals(redirectedUrl) - || (URL + anotherCurrentAccession).equals(redirectedUrl)); + || (URL + anotherCurrentAccession).equals(redirectedUrl)); // and then ResponseEntity>> getVariantsResponse = testRestTemplate @@ -783,7 +786,7 @@ public void testGetVariantsMergedSeveralTimes() assertEquals(HttpStatus.OK, getVariantsResponse.getStatusCode()); assertEquals(1, getVariantsResponse.getBody().size()); assertTrue(Arrays.asList(currentAccession, anotherCurrentAccession) - .contains(getVariantsResponse.getBody().get(0).getAccession())); + .contains(getVariantsResponse.getBody().get(0).getAccession())); assertClusteredVariantCreatedDateNotNull(getVariantsResponse.getBody()); } @@ -797,38 +800,38 @@ public void testGetMergedAndDeprecatedVariants() // given Long outdatedAccession = 1L; ClusteredVariant variant1 = new ClusteredVariant("ASMACC01", 2000, "CHROM1", 1234, VariantType.SNV, false, - null); + null); DbsnpClusteredVariantEntity clusteredVariantEntity1 = new DbsnpClusteredVariantEntity(outdatedAccession, - "hash-100", variant1, 1); + "hash-100", variant1, 1); ClusteredVariant variant2 = new ClusteredVariant("ASMACC02", 2000, "CHROM2", 1234, VariantType.SNV, false, - null); + null); DbsnpClusteredVariantEntity clusteredVariantEntity2 = new DbsnpClusteredVariantEntity(outdatedAccession, - "hash-200", variant2, 1); + "hash-200", variant2, 1); ClusteredVariant variant3 = new ClusteredVariant("ASMACC02", 2000, "CHROM2", 1234, VariantType.SNV, false, - null); + null); DbsnpClusteredVariantEntity clusteredVariantEntity3 = new DbsnpClusteredVariantEntity(outdatedAccession, - "hash-300", variant3, 1); + "hash-300", variant3, 1); Long currentAccession = 2L; ClusteredVariant variant4 = new ClusteredVariant("ASMACC02", 2000, "CHROM2", 1234, VariantType.SNV, false, - null); + null); DbsnpClusteredVariantEntity clusteredVariantEntity4 = new DbsnpClusteredVariantEntity(currentAccession, - "hash-400", variant4, 1); + "hash-400", variant4, 1); Long anotherCurrentAccession = 3L; ClusteredVariant variant5 = new ClusteredVariant("ASMACC01", 2000, "CHROM2", 1234, VariantType.SNV, false, - null); + null); DbsnpClusteredVariantEntity clusteredVariantEntity5 = new DbsnpClusteredVariantEntity(anotherCurrentAccession, - "hash-500", variant5, 1); + "hash-500", variant5, 1); mongoTemplate.dropCollection(DbsnpClusteredVariantEntity.class); mongoTemplate.insert(Arrays.asList(clusteredVariantEntity1, clusteredVariantEntity4, clusteredVariantEntity5), - DbsnpClusteredVariantEntity.class); + DbsnpClusteredVariantEntity.class); dbsnpService.merge(outdatedAccession, currentAccession, - "Just for testing the endpoint, let's pretend the variants are equivalent"); + "Just for testing the endpoint, let's pretend the variants are equivalent"); mongoTemplate.insert(Arrays.asList(clusteredVariantEntity2), DbsnpClusteredVariantEntity.class); dbsnpService.merge(outdatedAccession, anotherCurrentAccession, - "Second merge. This can totally happen importing from dbSNP. See rs106458077"); + "Second merge. This can totally happen importing from dbSNP. See rs106458077"); mongoTemplate.insert(Arrays.asList(clusteredVariantEntity3), DbsnpClusteredVariantEntity.class); dbsnpService.deprecate(outdatedAccession, "And then deprecated it."); @@ -899,18 +902,18 @@ public void testGetDeprecatedEvaClusteredVariant() // given Long deprecatedAccession = 1L; ClusteredVariant variant1 = new ClusteredVariant("ASMACC01", 2000, "CHROM1", 1234, VariantType.SNV, false, - null); + null); DbsnpClusteredVariantEntity clusteredVariantEntity1 = new DbsnpClusteredVariantEntity(deprecatedAccession, - "hash-100", variant1, 1); + "hash-100", variant1, 1); ClusteredVariant variant2 = new ClusteredVariant("ASMACC02", 2000, "CHROM2", 1234, VariantType.SNV, false, - null); + null); Long otherAccession = 2L; DbsnpClusteredVariantEntity clusteredVariantEntity2 = new DbsnpClusteredVariantEntity(otherAccession, - "hash-200", variant2, 1); + "hash-200", variant2, 1); mongoTemplate.dropCollection(DbsnpClusteredVariantEntity.class); mongoTemplate.insert(Arrays.asList(clusteredVariantEntity1, clusteredVariantEntity2), - DbsnpClusteredVariantEntity.class); + DbsnpClusteredVariantEntity.class); clusteredService.deprecate(deprecatedAccession, "deprecated for testing"); String getVariantUrl = URL + deprecatedAccession; @@ -929,10 +932,10 @@ public void testGetDeprecatedEvaClusteredVariant() public void testGetByIdFields() { ResponseEntity>> getVariantsResponse = controller.getByIdFields(clusteredVariantEntity1.getAssemblyAccession(), - clusteredVariantEntity1.getContig(), - clusteredVariantEntity1.getStart(), - clusteredVariantEntity1.getType(), - ContigNamingConvention.INSDC); + clusteredVariantEntity1.getContig(), + clusteredVariantEntity1.getStart(), + clusteredVariantEntity1.getType(), + ContigNamingConvention.INSDC); assertEquals(HttpStatus.OK, getVariantsResponse.getStatusCode()); assertEquals(clusteredVariantEntity1.getAccession(), getVariantsResponse.getBody().get(0).getAccession()); @@ -942,35 +945,35 @@ public void testGetByIdFields() { public void testGetByIdFields_withContigTranslation() { ResponseEntity>> getVariantsResponse = controller.getByIdFields(clusteredVariantEntity1.getAssemblyAccession(), - clusteredVariantEntity1.getContig() + ENA_CONTIG_SUFFIX, - clusteredVariantEntity1.getStart(), - clusteredVariantEntity1.getType(), - ContigNamingConvention.ENA_SEQUENCE_NAME); + clusteredVariantEntity1.getContig() + ENA_CONTIG_SUFFIX, + clusteredVariantEntity1.getStart(), + clusteredVariantEntity1.getType(), + ContigNamingConvention.ENA_SEQUENCE_NAME); assertEquals(HttpStatus.OK, getVariantsResponse.getStatusCode()); assertEquals(clusteredVariantEntity1.getAccession(), getVariantsResponse.getBody().get(0).getAccession()); assertEquals(clusteredVariantEntity1.getContig() + ENA_CONTIG_SUFFIX, - getVariantsResponse.getBody().get(0).getData().getContig()); + getVariantsResponse.getBody().get(0).getData().getContig()); } @Test public void testGetByIdFields_withWrongContigTranslation() { assertThrows(ResponseStatusException.class, - () -> controller.getByIdFields(clusteredVariantEntity1.getAssemblyAccession(), - clusteredVariantEntity1.getContig(), - clusteredVariantEntity1.getStart(), - clusteredVariantEntity1.getType(), - ContigNamingConvention.UCSC)); + () -> controller.getByIdFields(clusteredVariantEntity1.getAssemblyAccession(), + clusteredVariantEntity1.getContig(), + clusteredVariantEntity1.getStart(), + clusteredVariantEntity1.getType(), + ContigNamingConvention.UCSC)); } @Test public void testGetByIdFieldsHumanVariant() { ResponseEntity>> getVariantsResponse = controller.getByIdFields(clusteredHumanVariantEntity1.getAssemblyAccession(), - clusteredHumanVariantEntity1.getContig(), - clusteredHumanVariantEntity1.getStart(), - clusteredHumanVariantEntity1.getType(), - ContigNamingConvention.INSDC); + clusteredHumanVariantEntity1.getContig(), + clusteredHumanVariantEntity1.getStart(), + clusteredHumanVariantEntity1.getType(), + ContigNamingConvention.INSDC); assertEquals(HttpStatus.OK, getVariantsResponse.getStatusCode()); assertEquals(clusteredHumanVariantEntity1.getAccession(), getVariantsResponse.getBody().get(0).getAccession()); @@ -980,10 +983,10 @@ public void testGetByIdFieldsHumanVariant() { public void testGetByIdFieldsHumanVariantInOperations() { ResponseEntity>> getVariantsResponse = controller.getByIdFields(clusteredHumanVariantEntity3.getAssemblyAccession(), - clusteredHumanVariantEntity3.getContig(), - clusteredHumanVariantEntity3.getStart(), - clusteredHumanVariantEntity3.getType(), - ContigNamingConvention.INSDC); + clusteredHumanVariantEntity3.getContig(), + clusteredHumanVariantEntity3.getStart(), + clusteredHumanVariantEntity3.getType(), + ContigNamingConvention.INSDC); assertEquals(HttpStatus.OK, getVariantsResponse.getStatusCode()); assertEquals(clusteredHumanVariantEntity3.getAccession(), getVariantsResponse.getBody().get(0).getAccession()); @@ -993,10 +996,10 @@ public void testGetByIdFieldsHumanVariantInOperations() { public void testGetByIdFieldsHumanVariantDoesntExists() { ResponseEntity>> getVariantsResponse = controller.getByIdFields(clusteredHumanVariantEntity3.getAssemblyAccession(), - clusteredHumanVariantEntity3.getContig(), - 1L, - clusteredHumanVariantEntity3.getType(), - ContigNamingConvention.INSDC); + clusteredHumanVariantEntity3.getContig(), + 1L, + clusteredHumanVariantEntity3.getType(), + ContigNamingConvention.INSDC); assertEquals(HttpStatus.NOT_FOUND, getVariantsResponse.getStatusCode()); } @@ -1005,20 +1008,20 @@ public void testGetByIdFieldsHumanVariantDoesntExists() { public void testGetByIdFieldsClusteredVariantDoesntExists() { ResponseEntity>> getVariantsResponse = controller.getByIdFields(clusteredVariantEntity1.getAssemblyAccession(), - clusteredVariantEntity1.getContig(), - 123, - clusteredVariantEntity1.getType(), - ContigNamingConvention.INSDC); + clusteredVariantEntity1.getContig(), + 123, + clusteredVariantEntity1.getType(), + ContigNamingConvention.INSDC); assertEquals(HttpStatus.NOT_FOUND, getVariantsResponse.getStatusCode()); } - @Test(expected = RuntimeException.class) + @Test public void getByIdFieldstError500() { String assemblyId = "GCA_ERROR"; String chromosome = "CHROM1"; int start = 123; - mockController.getByIdFields(assemblyId, chromosome, start, VariantType.SNV, ContigNamingConvention.INSDC); + assertThrows(RuntimeException.class, () -> mockController.getByIdFields(assemblyId, chromosome, start, VariantType.SNV, ContigNamingConvention.INSDC)); } @Test @@ -1055,7 +1058,7 @@ private void assertDatasets(BeaconAlleleResponse beaconAlleleResponse) { private void assertEmbeddedAlleleRequest(BeaconAlleleResponse beaconAlleleResponse, DbsnpClusteredVariantEntity clusteredVariantEntity) { assertEmbeddedAlleleRequest(beaconAlleleResponse, clusteredVariantEntity.getAssemblyAccession(), - clusteredVariantEntity.getStart(), clusteredVariantEntity.getType()); + clusteredVariantEntity.getStart(), clusteredVariantEntity.getType()); } private void assertEmbeddedAlleleRequest(BeaconAlleleResponse beaconAlleleResponse, String assemblyId, @@ -1100,7 +1103,7 @@ public void doesVariantExistFalse() { assertFalse(beaconAlleleResponse.isExists()); assertNull(beaconAlleleResponse.getDatasetAlleleResponses()); assertEmbeddedAlleleRequest(beaconAlleleResponse, clusteredVariantEntity1.getAssemblyAccession(), 123L, - clusteredVariantEntity1.getType()); + clusteredVariantEntity1.getType()); } @Test @@ -1114,7 +1117,7 @@ public void doesVariantExistError500() { assertEquals(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, response.getStatus()); assertEquals(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, - (int) beaconAlleleResponse.getError().getErrorCode()); + (int) beaconAlleleResponse.getError().getErrorCode()); assertEmbeddedAlleleRequest(beaconAlleleResponse, assemblyId, start, VariantType.SNV); } } diff --git a/eva-accession-ws/src/test/java/uk/ac/ebi/eva/accession/ws/SubmittedVariantsRestControllerTest.java b/eva-accession-ws/src/test/java/uk/ac/ebi/eva/accession/ws/SubmittedVariantsRestControllerTest.java index 1b9ea0d5a..eda734d86 100644 --- a/eva-accession-ws/src/test/java/uk/ac/ebi/eva/accession/ws/SubmittedVariantsRestControllerTest.java +++ b/eva-accession-ws/src/test/java/uk/ac/ebi/eva/accession/ws/SubmittedVariantsRestControllerTest.java @@ -17,10 +17,11 @@ */ package uk.ac.ebi.eva.accession.ws; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; +import jakarta.servlet.http.HttpServletResponse; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.Mockito; import org.springframework.beans.factory.annotation.Autowired; @@ -36,7 +37,7 @@ import org.springframework.http.ResponseEntity; import org.springframework.mock.web.MockHttpServletResponse; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import org.springframework.web.server.ResponseStatusException; import uk.ac.ebi.ampt2d.commons.accession.core.exceptions.AccessionCouldNotBeGeneratedException; import uk.ac.ebi.ampt2d.commons.accession.core.exceptions.AccessionDeprecatedException; @@ -46,19 +47,19 @@ import uk.ac.ebi.ampt2d.commons.accession.core.models.GetOrCreateAccessionWrapper; import uk.ac.ebi.ampt2d.commons.accession.rest.controllers.BasicRestController; import uk.ac.ebi.ampt2d.commons.accession.rest.dto.AccessionResponseDTO; - +import uk.ac.ebi.eva.accession.core.configuration.nonhuman.SubmittedVariantAccessioningConfiguration; import uk.ac.ebi.eva.accession.core.contigalias.ContigAliasService; import uk.ac.ebi.eva.accession.core.model.ISubmittedVariant; import uk.ac.ebi.eva.accession.core.model.SubmittedVariant; -import uk.ac.ebi.eva.accession.core.service.nonhuman.SubmittedVariantAccessioningService; -import uk.ac.ebi.eva.accession.core.configuration.nonhuman.SubmittedVariantAccessioningConfiguration; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpSubmittedVariantEntity; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpSubmittedVariantOperationEntity; -import uk.ac.ebi.eva.accession.core.repository.nonhuman.eva.SubmittedVariantAccessioningRepository; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantOperationEntity; +import uk.ac.ebi.eva.accession.core.repository.nonhuman.eva.SubmittedVariantAccessioningRepository; +import uk.ac.ebi.eva.accession.core.service.nonhuman.SubmittedVariantAccessioningService; import uk.ac.ebi.eva.accession.core.service.nonhuman.dbsnp.DbsnpSubmittedVariantInactiveService; import uk.ac.ebi.eva.accession.core.service.nonhuman.dbsnp.DbsnpSubmittedVariantMonotonicAccessioningService; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.ws.dto.BeaconAlleleRequest; import uk.ac.ebi.eva.accession.ws.dto.BeaconAlleleResponse; import uk.ac.ebi.eva.accession.ws.rest.SubmittedVariantsRestController; @@ -66,18 +67,17 @@ import uk.ac.ebi.eva.accession.ws.test.NoContigTranslationArgumentMatcher; import uk.ac.ebi.eva.commons.core.models.contigalias.ContigNamingConvention; -import javax.servlet.http.HttpServletResponse; import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.NoSuchElementException; import java.util.stream.Collectors; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertThrows; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.argThat; @@ -88,11 +88,11 @@ import static uk.ac.ebi.eva.accession.core.model.ISubmittedVariant.DEFAULT_SUPPORTED_BY_EVIDENCE; import static uk.ac.ebi.eva.accession.core.model.ISubmittedVariant.DEFAULT_VALIDATED; -@RunWith(SpringRunner.class) +@ExtendWith(SpringExtension.class) @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) @Import({SubmittedVariantAccessioningConfiguration.class}) @TestPropertySource("classpath:accession-ws-test.properties") -public class SubmittedVariantsRestControllerTest { +public class SubmittedVariantsRestControllerTest extends MongoTestContainerHelper{ private static String TEST_APPLICATION_INSTANCE_ID = "test-application-instance-id"; private static final String URL = "/v1/submitted-variants/"; @@ -139,7 +139,7 @@ public class SubmittedVariantsRestControllerTest { private SubmittedVariant variant3; - @Before + @BeforeEach public void setUp() throws AccessionCouldNotBeGeneratedException { repository.deleteAll(); mongoTemplate.dropCollection(DbsnpSubmittedVariantEntity.class); @@ -155,8 +155,8 @@ public void setUp() throws AccessionCouldNotBeGeneratedException { SubmittedVariantsBeaconService mockSubmittedVariantsBeaconService = Mockito.spy(new SubmittedVariantsBeaconService(service)); Mockito.doThrow(new RuntimeException("Some unexpected error")).when(mockSubmittedVariantsBeaconService).queryBeacon(null, "alt", "ref", - "CHROM1", 1, "ref", - ContigNamingConvention.INSDC, false); + "CHROM1", 1, "ref", + ContigNamingConvention.INSDC, false); mockController = new SubmittedVariantsRestController(mockService, mockSubmittedVariantsBeaconService); setUpContigAliasMock(); @@ -166,20 +166,20 @@ private void setUpContigAliasMock() { NoContigTranslationArgumentMatcher contigMatcher = new NoContigTranslationArgumentMatcher(); when(contigAliasService.translateContigToInsdc(anyString(), anyString(), argThat(contigMatcher))) - .thenCallRealMethod(); + .thenCallRealMethod(); when(contigAliasService.translateContigToInsdc(anyString(), anyString(), eq(ContigNamingConvention.ENA_SEQUENCE_NAME))) - .then(invocation -> { - String contigName = invocation.getArgument(0); - if (contigName.endsWith(ENA_CONTIG_SUFFIX)) { - return contigName.substring(0, contigName.length() - ENA_CONTIG_SUFFIX.length()); - } - throw new NoSuchElementException("Tried to translate non-ENA contig using ENA naming convention"); - }); + .then(invocation -> { + String contigName = invocation.getArgument(0); + if (contigName.endsWith(ENA_CONTIG_SUFFIX)) { + return contigName.substring(0, contigName.length() - ENA_CONTIG_SUFFIX.length()); + } + throw new NoSuchElementException("Tried to translate non-ENA contig using ENA naming convention"); + }); when(contigAliasService.translateContigToInsdc(anyString(), anyString(), eq(ContigNamingConvention.UCSC))) .thenThrow(NoSuchElementException.class); when(contigAliasService.translateContigFromInsdc(anyString(), argThat(contigMatcher))) - .thenCallRealMethod(); + .thenCallRealMethod(); when(contigAliasService.translateContigFromInsdc(any(), eq(ContigNamingConvention.ENA_SEQUENCE_NAME))) .then(invocation -> { String contigName = invocation.getArgument(0); @@ -193,10 +193,10 @@ private void setUpContigAliasMock() { .thenCallRealMethod(); when(contigAliasService.createSubmittedVariantAccessionWrapperWithNewContig(any(), anyString())) - .thenCallRealMethod(); + .thenCallRealMethod(); } - @After + @AfterEach public void tearDown() { repository.deleteAll(); mongoTemplate.dropCollection(DbsnpSubmittedVariantEntity.class); @@ -293,12 +293,12 @@ public void testGetByIdFieldsSingleStudyPerRequest() { ISubmittedVariant variant = generatedAccession.getData(); ResponseEntity>> getVariantsResponse = controller.getByIdFields(variant.getReferenceSequenceAccession(), - variant.getContig(), - Collections.singletonList( - variant.getProjectAccession()), - variant.getStart(), variant.getReferenceAllele(), - variant.getAlternateAllele(), - ContigNamingConvention.INSDC); + variant.getContig(), + Collections.singletonList( + variant.getProjectAccession()), + variant.getStart(), variant.getReferenceAllele(), + variant.getAlternateAllele(), + ContigNamingConvention.INSDC); assertEquals(1, getVariantsResponse.getBody().size()); assertCreatedDateNotNull(getVariantsResponse.getBody()); @@ -309,15 +309,15 @@ public void testGetByIdFieldsSingleStudyPerRequest() { @Test public void testGetByIdFieldsMultipleStudiesPerRequest() { List multipleProjectAccessions = Arrays.asList(variant1.getProjectAccession(), - variant2.getProjectAccession(), - variant3.getProjectAccession()); + variant2.getProjectAccession(), + variant3.getProjectAccession()); ResponseEntity>> getVariantsResponse = controller.getByIdFields(variant2.getReferenceSequenceAccession(), - variant2.getContig(), multipleProjectAccessions, - variant2.getStart(), variant2.getReferenceAllele(), - variant2.getAlternateAllele(), - ContigNamingConvention.INSDC); + variant2.getContig(), multipleProjectAccessions, + variant2.getStart(), variant2.getReferenceAllele(), + variant2.getAlternateAllele(), + ContigNamingConvention.INSDC); assertEquals(2, getVariantsResponse.getBody().size()); assertCreatedDateNotNull(getVariantsResponse.getBody()); @@ -331,12 +331,12 @@ public void testGetByIdFields_withContigTranslation() { ISubmittedVariant variant = generatedAccession.getData(); ResponseEntity>> getVariantsResponse = controller.getByIdFields(variant.getReferenceSequenceAccession(), - variant.getContig() + ENA_CONTIG_SUFFIX, - Collections.singletonList( - variant.getProjectAccession()), - variant.getStart(), variant.getReferenceAllele(), - variant.getAlternateAllele(), - ContigNamingConvention.ENA_SEQUENCE_NAME); + variant.getContig() + ENA_CONTIG_SUFFIX, + Collections.singletonList( + variant.getProjectAccession()), + variant.getStart(), variant.getReferenceAllele(), + variant.getAlternateAllele(), + ContigNamingConvention.ENA_SEQUENCE_NAME); assertEquals(1, getVariantsResponse.getBody().size()); assertCreatedDateNotNull(getVariantsResponse.getBody()); @@ -348,13 +348,13 @@ public void testGetByIdFields_withContigTranslation() { @Test public void testGetByIdFields_withWrongContigTranslation() { assertThrows(ResponseStatusException.class, - () -> controller.getByIdFields(variant1.getReferenceSequenceAccession(), - variant1.getContig(), - Collections.singletonList(variant1.getProjectAccession()), - variant1.getStart(), - variant1.getReferenceAllele(), - variant1.getAlternateAllele(), - ContigNamingConvention.UCSC)); + () -> controller.getByIdFields(variant1.getReferenceSequenceAccession(), + variant1.getContig(), + Collections.singletonList(variant1.getProjectAccession()), + variant1.getStart(), + variant1.getReferenceAllele(), + variant1.getAlternateAllele(), + ContigNamingConvention.UCSC)); } @Test @@ -382,8 +382,8 @@ public void testDoesVariantExistFoundExistingVariantsSingleStudyPerRequest() { @Test public void testDoesVariantExistFoundExistingVariantsMultipleStudiesPerRequest() { List multipleProjectAccessions = Arrays.asList(variant1.getProjectAccession(), - variant2.getProjectAccession(), - variant3.getProjectAccession()); + variant2.getProjectAccession(), + variant3.getProjectAccession()); for (AccessionWrapper generatedAccession : generatedAccessions) { ISubmittedVariant variant = generatedAccession.getData(); @@ -435,7 +435,7 @@ public void testDoesVariantExistWith400Error() { assertEquals(HttpServletResponse.SC_BAD_REQUEST, response.getStatus()); assertEquals(HttpServletResponse.SC_BAD_REQUEST, beaconAlleleResponse.getError().getErrorCode()); assertEquals("Please provide a positive number as start position", - beaconAlleleResponse.getError().getErrorMessage()); + beaconAlleleResponse.getError().getErrorMessage()); BeaconAlleleRequest embeddedRequestObject = beaconAlleleResponse.getAlleleRequest(); assertEquals(variant1.getReferenceSequenceAccession(), embeddedRequestObject.getAssemblyId()); @@ -449,7 +449,7 @@ public void testDoesVariantExistWith400Error() { public void testDoesVariantExistWith500Error() { HttpServletResponse response = new MockHttpServletResponse(); BeaconAlleleResponse beaconAlleleResponse = mockController.doesVariantExist( - "asm", "CHROM1" + ENA_CONTIG_SUFFIX, null,1, "ref", "alt", response); + "asm", "CHROM1" + ENA_CONTIG_SUFFIX, null, 1, "ref", "alt", response); assertEquals(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, response.getStatus()); assertEquals(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, beaconAlleleResponse.getError().getErrorCode()); @@ -465,26 +465,26 @@ public void testDoesVariantExistWith500Error() { @Test public void testGetRedirectionForMergedVariants() throws AccessionCouldNotBeGeneratedException, AccessionMergedException, AccessionDoesNotExistException, - AccessionDeprecatedException { + AccessionDeprecatedException { // given Long CLUSTERED_VARIANT = null; SubmittedVariant variant1 = new SubmittedVariant("ASMACC01", 2000, "PROJACC01", "CHROM1", 1234, "REF", "ALT", - CLUSTERED_VARIANT); + CLUSTERED_VARIANT); SubmittedVariant variant2 = new SubmittedVariant("ASMACC02", 2000, "PROJACC02", "CHROM2", 1234, "REF", "ALT", - CLUSTERED_VARIANT); + CLUSTERED_VARIANT); List> accessions = service.getOrCreate( Arrays.asList(variant1, variant2), TEST_APPLICATION_INSTANCE_ID); Long outdatedAccession = accessions.get(0).getAccession(); Long currentAccession = accessions.get(1).getAccession(); service.merge(outdatedAccession, - currentAccession, - "Just for testing the endpoint, let's pretend the variants are equivalent"); + currentAccession, + "Just for testing the endpoint, let's pretend the variants are equivalent"); // when String getVariantsUrl = URL + outdatedAccession; ResponseEntity firstResponse = testRestTemplate.exchange(getVariantsUrl, HttpMethod.GET, null, - String.class); + String.class); // then assertEquals(HttpStatus.MOVED_PERMANENTLY, firstResponse.getStatusCode()); @@ -507,7 +507,7 @@ public void testGetRedirectionForMergedVariants() /** * If there is a variant with operations of several types (e.g. UPDATED and MERGED) the MERGED event should take * priority and the endpoint should return a redirection, **even if the MERGED event is not the last one**. - * + *

* Example dbsnp variant: ss825691104. It was declustered from rs796064771 and merged into ss825691103 at the same * time. */ @@ -517,33 +517,33 @@ public void testGetRedirectionForMergedAndUpdatedVariants() // given Long CLUSTERED_VARIANT = null; SubmittedVariant variant1 = new SubmittedVariant("ASMACC01", 2000, "PROJACC01", "CHROM1", 1234, "REF", "ALT", - CLUSTERED_VARIANT); + CLUSTERED_VARIANT); Long outdatedAccession = 1L; SubmittedVariantEntity submittedVariantEntity1 = new SubmittedVariantEntity(outdatedAccession, "hash-100", - variant1, 1); + variant1, 1); Long currentAccession = 2L; SubmittedVariant variant2 = new SubmittedVariant("ASMACC02", 2000, "PROJACC02", "CHROM2", 1234, "REF", "ALT", - CLUSTERED_VARIANT); + CLUSTERED_VARIANT); SubmittedVariantEntity submittedVariantEntity2 = new SubmittedVariantEntity(currentAccession, "hash-200", - variant2, 1); + variant2, 1); mongoTemplate.insert(Arrays.asList(submittedVariantEntity1, submittedVariantEntity2), - DbsnpSubmittedVariantEntity.class); + DbsnpSubmittedVariantEntity.class); dbsnpService.merge(outdatedAccession, - currentAccession, - "Just for testing the endpoint, let's pretend the variants are equivalent"); + currentAccession, + "Just for testing the endpoint, let's pretend the variants are equivalent"); SubmittedVariant updatedVariant = new SubmittedVariant(variant1); updatedVariant.setContig("contig_2"); dbsnpInactiveService.update(new DbsnpSubmittedVariantEntity(outdatedAccession, "hash-300", updatedVariant, 1), - "update a merged variant"); + "update a merged variant"); // when String getVariantsUrl = URL + outdatedAccession; ResponseEntity firstResponse = testRestTemplate.exchange(getVariantsUrl, HttpMethod.GET, null, - String.class); + String.class); // then assertEquals(HttpStatus.MOVED_PERMANENTLY, firstResponse.getStatusCode()); @@ -566,10 +566,10 @@ public void testGetRedirectionForMergedAndUpdatedVariants() @Test public void testGetDeprecatedEvaSubmittedVariant() throws AccessionMergedException, AccessionDoesNotExistException, - AccessionDeprecatedException { + AccessionDeprecatedException { // given Long accession = generatedAccessions.stream().filter(wrapper -> wrapper.getData().equals(variant1)) - .findFirst().get().getAccession(); + .findFirst().get().getAccession(); service.deprecate(accession, "deprecated for testing"); String getVariantUrl = URL + accession; @@ -590,18 +590,18 @@ public void testGetDeprecatedDbsnpSubmittedVariant() // given Long CLUSTERED_VARIANT = null; SubmittedVariant variant1 = new SubmittedVariant("ASMACC01", 2000, "PROJACC01", "CHROM1", 1234, "REF", "ALT", - CLUSTERED_VARIANT); + CLUSTERED_VARIANT); Long deprecatedAccession = 1L; SubmittedVariantEntity submittedVariantEntity1 = new SubmittedVariantEntity(deprecatedAccession, "hash-100", - variant1, 1); + variant1, 1); Long otherAccession = 2L; SubmittedVariant variant2 = new SubmittedVariant("ASMACC02", 2000, "PROJACC02", "CHROM2", 1234, "REF", "ALT", - CLUSTERED_VARIANT); + CLUSTERED_VARIANT); SubmittedVariantEntity submittedVariantEntity2 = new SubmittedVariantEntity(otherAccession, "hash-200", - variant2, 1); + variant2, 1); mongoTemplate.insert(Arrays.asList(submittedVariantEntity1, submittedVariantEntity2), - DbsnpSubmittedVariantEntity.class); + DbsnpSubmittedVariantEntity.class); service.deprecate(deprecatedAccession, "deprecated for testing"); String getVariantUrl = URL + deprecatedAccession; diff --git a/eva-accession-ws/src/test/java/uk/ac/ebi/eva/accession/ws/test/MongoTestConfiguration.java b/eva-accession-ws/src/test/java/uk/ac/ebi/eva/accession/ws/test/MongoTestConfiguration.java new file mode 100644 index 000000000..5a4751e52 --- /dev/null +++ b/eva-accession-ws/src/test/java/uk/ac/ebi/eva/accession/ws/test/MongoTestConfiguration.java @@ -0,0 +1,105 @@ +/* + * Copyright 2021 EMBL - European Bioinformatics Institute + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package uk.ac.ebi.eva.accession.ws.test; + +import com.mongodb.ReadPreference; +import com.mongodb.WriteConcern; +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoClients; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.domain.EntityScan; +import org.springframework.boot.test.autoconfigure.data.mongo.AutoConfigureDataMongo; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Primary; +import org.springframework.data.mongodb.MongoDatabaseFactory; +import org.springframework.data.mongodb.config.EnableMongoAuditing; +import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.data.mongodb.core.SimpleMongoClientDatabaseFactory; +import org.springframework.data.mongodb.core.WriteResultChecking; +import org.springframework.data.mongodb.core.convert.DefaultDbRefResolver; +import org.springframework.data.mongodb.core.convert.DefaultMongoTypeMapper; +import org.springframework.data.mongodb.core.convert.MappingMongoConverter; +import org.springframework.data.mongodb.core.convert.MongoCustomConversions; +import org.springframework.data.mongodb.core.mapping.MongoMappingContext; +import org.springframework.data.mongodb.repository.config.EnableMongoRepositories; + +import java.util.Collections; + +@Configuration +@EntityScan(basePackages = {"uk.ac.ebi.eva.accession.core.persistence"}) +@EnableMongoRepositories(basePackages = { + "uk.ac.ebi.eva.accession.core.persistence", + "uk.ac.ebi.ampt2d.commons.accession.persistence.mongodb.repository"}) +@EnableMongoAuditing +@AutoConfigureDataMongo +public class MongoTestConfiguration { + + @Value("${spring.data.mongodb.uri}") + private String mongoUri; + + @Value("${spring.data.mongodb.database}") + private String database; + + @Value("${human.mongodb.database}") + private String humanDatabase; + + @Value("${mongodb.read-preference:primary}") + private String readPreference; + + @Bean + @Primary + public MongoClient mongoClient() { + return MongoClients.create(mongoUri); + } + + @Bean("primaryFactory") + @Primary + public MongoDatabaseFactory mongoDbFactory() { + return new SimpleMongoClientDatabaseFactory(mongoClient(), database); + } + + @Bean + @Primary + public MongoCustomConversions mongoCustomConversions() { + return new MongoCustomConversions(Collections.emptyList()); + } + + @Bean + @Primary + public MongoMappingContext mongoMappingContext(MongoCustomConversions conversions) { + MongoMappingContext context = new MongoMappingContext(); + context.setSimpleTypeHolder(conversions.getSimpleTypeHolder()); + context.setAutoIndexCreation(false); + return context; + } + + @Bean + @Primary + public MongoTemplate mongoTemplate(MongoMappingContext mongoMappingContext, + MongoCustomConversions conversions) { + MappingMongoConverter converter = new MappingMongoConverter( + new DefaultDbRefResolver(mongoDbFactory()), mongoMappingContext); + converter.setCustomConversions(conversions); + converter.setTypeMapper(new DefaultMongoTypeMapper(null)); + converter.afterPropertiesSet(); + MongoTemplate mongoTemplate = new MongoTemplate(mongoDbFactory(), converter); + mongoTemplate.setReadPreference(ReadPreference.valueOf(readPreference)); + mongoTemplate.setWriteConcern(WriteConcern.MAJORITY); + mongoTemplate.setWriteResultChecking(WriteResultChecking.EXCEPTION); + return mongoTemplate; + } +} diff --git a/eva-accession-ws/src/test/java/uk/ac/ebi/eva/accession/ws/test/TestConfiguration.java b/eva-accession-ws/src/test/java/uk/ac/ebi/eva/accession/ws/test/TestConfiguration.java index b4ede1ea3..998c549a4 100644 --- a/eva-accession-ws/src/test/java/uk/ac/ebi/eva/accession/ws/test/TestConfiguration.java +++ b/eva-accession-ws/src/test/java/uk/ac/ebi/eva/accession/ws/test/TestConfiguration.java @@ -20,13 +20,12 @@ import org.springframework.context.annotation.Configuration; import uk.ac.ebi.ampt2d.commons.accession.core.AccessionSaveMode; import uk.ac.ebi.ampt2d.commons.accession.hashing.SHA1HashingFunction; - +import uk.ac.ebi.eva.accession.core.generators.DbsnpMonotonicAccessionGenerator; import uk.ac.ebi.eva.accession.core.model.IClusteredVariant; import uk.ac.ebi.eva.accession.core.model.ISubmittedVariant; import uk.ac.ebi.eva.accession.core.service.nonhuman.dbsnp.DbsnpClusteredVariantAccessioningDatabaseService; -import uk.ac.ebi.eva.accession.core.generators.DbsnpMonotonicAccessionGenerator; -import uk.ac.ebi.eva.accession.core.service.nonhuman.dbsnp.DbsnpSubmittedVariantAccessioningDatabaseService; import uk.ac.ebi.eva.accession.core.service.nonhuman.dbsnp.DbsnpClusteredVariantMonotonicAccessioningService; +import uk.ac.ebi.eva.accession.core.service.nonhuman.dbsnp.DbsnpSubmittedVariantAccessioningDatabaseService; import uk.ac.ebi.eva.accession.core.service.nonhuman.dbsnp.DbsnpSubmittedVariantMonotonicAccessioningService; import uk.ac.ebi.eva.accession.core.summary.ClusteredVariantSummaryFunction; import uk.ac.ebi.eva.accession.core.summary.SubmittedVariantSummaryFunction; diff --git a/eva-accession-ws/src/test/java/uk/ac/ebi/eva/accession/ws/test/rule/FixSpringMongoDbRule.java b/eva-accession-ws/src/test/java/uk/ac/ebi/eva/accession/ws/test/rule/FixSpringMongoDbRule.java deleted file mode 100644 index b3e08f7a5..000000000 --- a/eva-accession-ws/src/test/java/uk/ac/ebi/eva/accession/ws/test/rule/FixSpringMongoDbRule.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * - * Copyright 2021 EMBL - European Bioinformatics Institute - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -package uk.ac.ebi.eva.accession.ws.test.rule; - -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfiguration; -import com.lordofthejars.nosqlunit.mongodb.SpringMongoDbRule; - - -/** - * Temporary fix until nosql unit rc-6 or final is released - */ -public class FixSpringMongoDbRule extends SpringMongoDbRule { - - public FixSpringMongoDbRule(MongoDbConfiguration mongoDbConfiguration) { - super(mongoDbConfiguration); - } - - public FixSpringMongoDbRule(MongoDbConfiguration mongoDbConfiguration, Object object) { - super(mongoDbConfiguration, object); - } - - @Override - public void close() { - // DO NOT CLOSE the connection (Spring will do it when destroying the context) - } -} diff --git a/eva-accession-ws/src/test/resources/accession-ws-test.properties b/eva-accession-ws/src/test/resources/accession-ws-test.properties index 3cf1bcdc1..82e3e9615 100644 --- a/eva-accession-ws/src/test/resources/accession-ws-test.properties +++ b/eva-accession-ws/src/test/resources/accession-ws-test.properties @@ -14,13 +14,10 @@ accessioning.monotonic.test-rs.nextBlockInterval=1000000000 # this will be replaced with a default value "localhost" when run locally # (due to the filtering attribute specified in this project's POM) # (OR) this variable can be overridden in Gitlab with "mongo" (Gitlab services are exposed under their own hostname) -spring.data.mongodb.uri=mongodb://|eva.mongo.host.test|:27017 spring.data.mongodb.database=eva-accession-ws-test-db # See https://github.com/spring-projects/spring-boot/wiki/Spring-Boot-2.1-Release-Notes#bean-overriding spring.main.allow-bean-definition-overriding=true mongodb.read-preference=primary - -human.mongodb.uri=mongodb://|eva.mongo.host.test|:27017 human.mongodb.database=eva-accession-ws-test-db-human eva.api.base-url=http://localhost diff --git a/eva-remapping-get-source/pom.xml b/eva-remapping-get-source/pom.xml index 6c3b5d198..a01e9bc5c 100644 --- a/eva-remapping-get-source/pom.xml +++ b/eva-remapping-get-source/pom.xml @@ -2,12 +2,13 @@ + 4.0.0 + eva-accession uk.ac.ebi.eva - 0.6.58-SNAPSHOT + ${revision} - 4.0.0 eva-remapping-get-source jar @@ -24,8 +25,8 @@ uk.ac.ebi.eva eva-accession-core + ${revision} test-jar - ${project.version} test @@ -48,8 +49,18 @@ test - com.lordofthejars - nosqlunit-mongodb + org.springframework.boot + spring-boot-testcontainers + test + + + org.testcontainers + mongodb + test + + + org.testcontainers + junit-jupiter test @@ -90,8 +101,4 @@ - - localhost - - diff --git a/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/batch/io/DbsnpSubmittedVariantMongoReader.java b/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/batch/io/DbsnpSubmittedVariantMongoReader.java index c3e98cfcc..648edf5a7 100644 --- a/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/batch/io/DbsnpSubmittedVariantMongoReader.java +++ b/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/batch/io/DbsnpSubmittedVariantMongoReader.java @@ -15,7 +15,6 @@ */ package uk.ac.ebi.eva.remapping.source.batch.io; -import org.apache.commons.collections.CollectionUtils; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.data.mongodb.core.query.Criteria; import org.springframework.data.mongodb.core.query.Meta; @@ -39,14 +38,14 @@ public class DbsnpSubmittedVariantMongoReader extends MongoDbCursorItemReader projects, int taxonomy) { + List projects, int taxonomy) { setMongoTemplate(mongoTemplate); setTargetType(DbsnpSubmittedVariantEntity.class); Criteria criteria = where(REFERENCE_SEQUENCE_FIELD).is(assemblyAccession); criteria.and(REMAPPEDFROM_KEY).exists(false); - if (!CollectionUtils.isEmpty(projects)) { + if (projects != null && !projects.isEmpty()) { criteria.and(PROJECT_KEY).in(projects); } diff --git a/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/batch/io/EvaSubmittedVariantMongoReader.java b/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/batch/io/EvaSubmittedVariantMongoReader.java index 6e64de229..1c4707708 100644 --- a/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/batch/io/EvaSubmittedVariantMongoReader.java +++ b/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/batch/io/EvaSubmittedVariantMongoReader.java @@ -15,7 +15,6 @@ */ package uk.ac.ebi.eva.remapping.source.batch.io; -import org.apache.commons.collections.CollectionUtils; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.data.mongodb.core.query.Criteria; import org.springframework.data.mongodb.core.query.Meta; @@ -46,7 +45,7 @@ public EvaSubmittedVariantMongoReader(String assemblyAccession, MongoTemplate mo Criteria criteria = where(REFERENCE_SEQUENCE_FIELD).is(assemblyAccession); criteria.and(REMAPPEDFROM_KEY).exists(false); - if (!CollectionUtils.isEmpty(projects)) { + if (projects != null && !projects.isEmpty()) { criteria.and(PROJECT_KEY).in(projects); } diff --git a/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/batch/io/VariantContextWriter.java b/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/batch/io/VariantContextWriter.java index 52e7a03b9..a186d72d4 100644 --- a/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/batch/io/VariantContextWriter.java +++ b/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/batch/io/VariantContextWriter.java @@ -24,6 +24,7 @@ import htsjdk.variant.vcf.VCFInfoHeaderLine; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.batch.item.Chunk; import org.springframework.batch.item.ExecutionContext; import org.springframework.batch.item.ItemStreamException; import org.springframework.batch.item.ItemStreamWriter; @@ -31,7 +32,6 @@ import java.io.File; import java.nio.file.Path; import java.util.HashSet; -import java.util.List; import java.util.Set; /** @@ -87,15 +87,15 @@ protected Set buildHeaderLines() { metaData.add(new VCFInfoHeaderLine(SS_HASH, 1, VCFHeaderLineType.String, "Hash (_id in MongoDB) of the Submitted Variant from which the remapped variant was created")); metaData.add(new VCFInfoHeaderLine(RS_KEY, 1, VCFHeaderLineType.String, - "RS ID where this SS ID is clustered")); + "RS ID where this SS ID is clustered")); metaData.add(new VCFInfoHeaderLine(BACKPROP_RS_KEY, 1, VCFHeaderLineType.String, - "RS ID that was backpropagated to this SS from another remapping")); + "RS ID that was backpropagated to this SS from another remapping")); metaData.add(new VCFInfoHeaderLine(PROJECT_KEY, 1, VCFHeaderLineType.String, - "PROJECT ID associated with this SS ID")); + "PROJECT ID associated with this SS ID")); metaData.add(new VCFInfoHeaderLine(TAXONOMY, 1, VCFHeaderLineType.String, - "TAXONOMY ID associated with this SS ID")); + "TAXONOMY ID associated with this SS ID")); metaData.add(new VCFInfoHeaderLine(CREATED_DATE, 1, VCFHeaderLineType.String, - "Date when the SS ID was created")); + "Date when the SS ID was created")); return metaData; } @@ -104,7 +104,7 @@ private String getReferenceAssemblyLine() { } @Override - public void write(List variantContexts) throws Exception { + public void write(Chunk variantContexts) throws Exception { for (VariantContext variantContext : variantContexts) { writer.add(variantContext); } diff --git a/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/batch/processors/SubmittedVariantToVariantContextProcessor.java b/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/batch/processors/SubmittedVariantToVariantContextProcessor.java index df8e12025..44f08ee8a 100644 --- a/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/batch/processors/SubmittedVariantToVariantContextProcessor.java +++ b/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/batch/processors/SubmittedVariantToVariantContextProcessor.java @@ -19,7 +19,6 @@ import htsjdk.variant.variantcontext.VariantContext; import htsjdk.variant.variantcontext.VariantContextBuilder; import org.springframework.batch.item.ItemProcessor; - import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; import java.util.HashMap; @@ -27,9 +26,9 @@ import static uk.ac.ebi.eva.remapping.source.batch.io.VariantContextWriter.BACKPROP_RS_KEY; import static uk.ac.ebi.eva.remapping.source.batch.io.VariantContextWriter.CREATED_DATE; +import static uk.ac.ebi.eva.remapping.source.batch.io.VariantContextWriter.PROJECT_KEY; import static uk.ac.ebi.eva.remapping.source.batch.io.VariantContextWriter.RS_KEY; import static uk.ac.ebi.eva.remapping.source.batch.io.VariantContextWriter.SS_HASH; -import static uk.ac.ebi.eva.remapping.source.batch.io.VariantContextWriter.PROJECT_KEY; import static uk.ac.ebi.eva.remapping.source.batch.io.VariantContextWriter.TAXONOMY; /** diff --git a/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/configuration/RunnerConfiguration.java b/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/configuration/RunnerConfiguration.java index 1542410ed..01d53cd52 100644 --- a/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/configuration/RunnerConfiguration.java +++ b/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/configuration/RunnerConfiguration.java @@ -16,16 +16,10 @@ */ package uk.ac.ebi.eva.remapping.source.configuration; -import org.springframework.batch.core.configuration.annotation.BatchConfigurer; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; - -import uk.ac.ebi.eva.commons.batch.configuration.SpringBoot1CompatibilityConfiguration; import uk.ac.ebi.eva.commons.batch.job.JobExecutionApplicationListener; -import javax.persistence.EntityManagerFactory; -import javax.sql.DataSource; - @Configuration public class RunnerConfiguration { @@ -33,11 +27,4 @@ public class RunnerConfiguration { public JobExecutionApplicationListener jobExecutionApplicationListener() { return new JobExecutionApplicationListener(); } - - @Bean - public BatchConfigurer configurer(DataSource dataSource, EntityManagerFactory entityManagerFactory) - throws Exception { - return SpringBoot1CompatibilityConfiguration.getSpringBoot1CompatibleBatchConfigurer(dataSource, - entityManagerFactory); - } } \ No newline at end of file diff --git a/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/configuration/batch/io/VariantContextWriterConfiguration.java b/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/configuration/batch/io/VariantContextWriterConfiguration.java index c96348cab..e751d5d57 100644 --- a/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/configuration/batch/io/VariantContextWriterConfiguration.java +++ b/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/configuration/batch/io/VariantContextWriterConfiguration.java @@ -18,11 +18,10 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; - import uk.ac.ebi.eva.remapping.source.batch.io.VariantContextWriter; +import uk.ac.ebi.eva.remapping.source.configuration.BeanNames; import uk.ac.ebi.eva.remapping.source.parameters.InputParameters; import uk.ac.ebi.eva.remapping.source.parameters.ReportPathResolver; -import uk.ac.ebi.eva.remapping.source.configuration.BeanNames; import java.nio.file.Path; diff --git a/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/configuration/batch/jobs/ExportSubmittedVariantsJobConfiguration.java b/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/configuration/batch/jobs/ExportSubmittedVariantsJobConfiguration.java index ecd852e4e..c6a982d6a 100644 --- a/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/configuration/batch/jobs/ExportSubmittedVariantsJobConfiguration.java +++ b/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/configuration/batch/jobs/ExportSubmittedVariantsJobConfiguration.java @@ -19,16 +19,16 @@ import org.springframework.batch.core.Job; import org.springframework.batch.core.Step; import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; -import org.springframework.batch.core.configuration.annotation.JobBuilderFactory; +import org.springframework.batch.core.job.builder.JobBuilder; import org.springframework.batch.core.launch.support.RunIdIncrementer; +import org.springframework.batch.core.repository.JobRepository; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Import; - -import uk.ac.ebi.eva.remapping.source.configuration.batch.steps.ExportSubmittedVariantsStepConfiguration; import uk.ac.ebi.eva.remapping.source.configuration.BeanNames; +import uk.ac.ebi.eva.remapping.source.configuration.batch.steps.ExportSubmittedVariantsStepConfiguration; @Configuration @EnableBatchProcessing @@ -37,14 +37,14 @@ public class ExportSubmittedVariantsJobConfiguration { @Bean(BeanNames.EXPORT_SUBMITTED_VARIANTS_JOB) public Job accessionReleaseJob( - JobBuilderFactory jobBuilderFactory, + JobRepository jobRepository, @Autowired @Qualifier(BeanNames.EXPORT_EVA_SUBMITTED_VARIANTS_STEP) Step exportEvaSubmittedVariantsStep, @Autowired @Qualifier(BeanNames.EXPORT_DBSNP_SUBMITTED_VARIANTS_STEP) Step exportDbsnpSubmittedVariantsStep ) { - return jobBuilderFactory.get(BeanNames.EXPORT_SUBMITTED_VARIANTS_JOB) - .incrementer(new RunIdIncrementer()) - .start(exportEvaSubmittedVariantsStep) - .next(exportDbsnpSubmittedVariantsStep) - .build(); + return new JobBuilder(BeanNames.EXPORT_SUBMITTED_VARIANTS_JOB, jobRepository) + .incrementer(new RunIdIncrementer()) + .start(exportEvaSubmittedVariantsStep) + .next(exportDbsnpSubmittedVariantsStep) + .build(); } } diff --git a/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/configuration/batch/listeners/ListenersConfiguration.java b/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/configuration/batch/listeners/ListenersConfiguration.java index 4078594dd..47563cb45 100644 --- a/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/configuration/batch/listeners/ListenersConfiguration.java +++ b/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/configuration/batch/listeners/ListenersConfiguration.java @@ -20,12 +20,11 @@ import org.springframework.batch.core.listener.StepListenerSupport; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; - -import uk.ac.ebi.eva.remapping.source.batch.listeners.ExcludeVariantsListener; import uk.ac.ebi.eva.accession.core.batch.listeners.GenericProgressListener; -import uk.ac.ebi.eva.remapping.source.parameters.InputParameters; import uk.ac.ebi.eva.commons.core.models.pipeline.Variant; +import uk.ac.ebi.eva.remapping.source.batch.listeners.ExcludeVariantsListener; import uk.ac.ebi.eva.remapping.source.configuration.BeanNames; +import uk.ac.ebi.eva.remapping.source.parameters.InputParameters; @Configuration public class ListenersConfiguration { diff --git a/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/configuration/batch/steps/ExportSubmittedVariantsStepConfiguration.java b/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/configuration/batch/steps/ExportSubmittedVariantsStepConfiguration.java index be62e34e9..14af3acba 100644 --- a/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/configuration/batch/steps/ExportSubmittedVariantsStepConfiguration.java +++ b/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/configuration/batch/steps/ExportSubmittedVariantsStepConfiguration.java @@ -18,7 +18,8 @@ import htsjdk.variant.variantcontext.VariantContext; import org.springframework.batch.core.Step; import org.springframework.batch.core.StepExecutionListener; -import org.springframework.batch.core.configuration.annotation.StepBuilderFactory; +import org.springframework.batch.core.repository.JobRepository; +import org.springframework.batch.core.step.builder.StepBuilder; import org.springframework.batch.core.step.tasklet.TaskletStep; import org.springframework.batch.item.ItemProcessor; import org.springframework.batch.item.ItemReader; @@ -29,16 +30,16 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Import; - +import org.springframework.transaction.PlatformTransactionManager; +import uk.ac.ebi.eva.accession.core.batch.policies.IllegalStartSkipPolicy; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpSubmittedVariantEntity; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; -import uk.ac.ebi.eva.accession.core.batch.policies.IllegalStartSkipPolicy; +import uk.ac.ebi.eva.remapping.source.configuration.BeanNames; import uk.ac.ebi.eva.remapping.source.configuration.batch.io.SubmittedVariantMongoReaderConfiguration; import uk.ac.ebi.eva.remapping.source.configuration.batch.io.VariantContextWriterConfiguration; import uk.ac.ebi.eva.remapping.source.configuration.batch.listeners.ListenersConfiguration; import uk.ac.ebi.eva.remapping.source.configuration.batch.policies.PoliciesConfiguration; import uk.ac.ebi.eva.remapping.source.configuration.batch.processors.SubmittedVariantsProcessorConfiguration; -import uk.ac.ebi.eva.remapping.source.configuration.BeanNames; @Configuration @Import({SubmittedVariantMongoReaderConfiguration.class, @@ -50,7 +51,7 @@ public class ExportSubmittedVariantsStepConfiguration { @Bean(BeanNames.EXPORT_EVA_SUBMITTED_VARIANTS_STEP) public Step exportEvaSubmittedVariantsStep( - StepBuilderFactory stepBuilderFactory, + JobRepository jobRepository, PlatformTransactionManager transactionManager, SimpleCompletionPolicy chunkSizeCompletionPolicy, @Autowired @Qualifier(BeanNames.EVA_SUBMITTED_VARIANT_READER) ItemReader variantReader, @Autowired @Qualifier(BeanNames.SUBMITTED_VARIANT_PROCESSOR) ItemProcessor variantProcessor, @@ -58,8 +59,8 @@ public Step exportEvaSubmittedVariantsStep( @Autowired @Qualifier(BeanNames.PROGRESS_LISTENER) StepExecutionListener progressListener, @Autowired @Qualifier(BeanNames.EXCLUDE_VARIANTS_LISTENER) StepExecutionListener excludeVariantsListener, @Autowired IllegalStartSkipPolicy illegalStartSkipPolicy) { - TaskletStep step = stepBuilderFactory.get(BeanNames.EXPORT_EVA_SUBMITTED_VARIANTS_STEP) - .chunk(chunkSizeCompletionPolicy) + TaskletStep step = new StepBuilder(BeanNames.EXPORT_EVA_SUBMITTED_VARIANTS_STEP, jobRepository) + .chunk(chunkSizeCompletionPolicy, transactionManager) .reader(variantReader) .processor(variantProcessor) .writer(accessionWriter) @@ -73,7 +74,7 @@ public Step exportEvaSubmittedVariantsStep( @Bean(BeanNames.EXPORT_DBSNP_SUBMITTED_VARIANTS_STEP) public Step exportDbsnpSubmittedVariantsStep( - StepBuilderFactory stepBuilderFactory, + JobRepository jobRepository, PlatformTransactionManager transactionManager, SimpleCompletionPolicy chunkSizeCompletionPolicy, @Autowired @Qualifier(BeanNames.DBSNP_SUBMITTED_VARIANT_READER) ItemReader variantReader, @Autowired @Qualifier(BeanNames.SUBMITTED_VARIANT_PROCESSOR) ItemProcessor variantProcessor, @@ -81,8 +82,8 @@ public Step exportDbsnpSubmittedVariantsStep( @Autowired @Qualifier(BeanNames.PROGRESS_LISTENER) StepExecutionListener progressListener, @Autowired @Qualifier(BeanNames.EXCLUDE_VARIANTS_LISTENER) StepExecutionListener excludeVariantsListener, @Autowired IllegalStartSkipPolicy illegalStartSkipPolicy) { - TaskletStep step = stepBuilderFactory.get(BeanNames.EXPORT_DBSNP_SUBMITTED_VARIANTS_STEP) - .chunk(chunkSizeCompletionPolicy) + TaskletStep step = new StepBuilder(BeanNames.EXPORT_DBSNP_SUBMITTED_VARIANTS_STEP, jobRepository) + .chunk(chunkSizeCompletionPolicy, transactionManager) .reader(variantReader) .processor(variantProcessor) .writer(accessionWriter) diff --git a/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/parameters/InputParameters.java b/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/parameters/InputParameters.java index f64e8b77a..4f2d34406 100644 --- a/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/parameters/InputParameters.java +++ b/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/parameters/InputParameters.java @@ -16,7 +16,6 @@ package uk.ac.ebi.eva.remapping.source.parameters; -import org.apache.commons.collections.CollectionUtils; import org.springframework.batch.core.JobParameters; import org.springframework.batch.core.JobParametersBuilder; @@ -46,8 +45,8 @@ public JobParameters toJobParameters() { .addString("fasta", fasta) .addString("assemblyReportUrl", assemblyReportUrl) .addString("outputFolder", outputFolder) - .addString("projects", CollectionUtils.isEmpty(projects) ? "" : String.join(",", projects)) - .addLong("taxonomy", (long)taxonomy) + .addString("projects", (projects == null || projects.isEmpty()) ? "" : String.join(",", projects)) + .addLong("taxonomy", (long) taxonomy) .toJobParameters(); } diff --git a/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/runner/AccessionRemappingJobLauncherCommandLineRunner.java b/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/runner/AccessionRemappingJobLauncherCommandLineRunner.java index 70e6ea907..ee20d70ae 100644 --- a/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/runner/AccessionRemappingJobLauncherCommandLineRunner.java +++ b/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/runner/AccessionRemappingJobLauncherCommandLineRunner.java @@ -24,7 +24,6 @@ import org.springframework.batch.core.JobParametersInvalidException; import org.springframework.batch.core.explore.JobExplorer; import org.springframework.batch.core.launch.JobLauncher; -import org.springframework.batch.core.launch.JobParametersNotFoundException; import org.springframework.batch.core.repository.JobExecutionAlreadyRunningException; import org.springframework.batch.core.repository.JobInstanceAlreadyCompleteException; import org.springframework.batch.core.repository.JobRepository; @@ -32,24 +31,23 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.ExitCodeGenerator; -import org.springframework.boot.autoconfigure.batch.JobLauncherCommandLineRunner; +import org.springframework.boot.autoconfigure.batch.JobLauncherApplicationRunner; import org.springframework.context.ApplicationEventPublisherAware; import org.springframework.stereotype.Component; import org.springframework.util.PatternMatchUtils; - import uk.ac.ebi.eva.accession.core.runner.CommandLineRunnerUtils; -import uk.ac.ebi.eva.remapping.source.parameters.InputParameters; import uk.ac.ebi.eva.commons.batch.exception.NoJobToExecuteException; import uk.ac.ebi.eva.commons.batch.exception.NoParametersHaveBeenPassedException; import uk.ac.ebi.eva.commons.batch.exception.NoPreviousJobExecutionException; import uk.ac.ebi.eva.commons.batch.exception.UnknownJobException; import uk.ac.ebi.eva.commons.batch.job.JobExecutionApplicationListener; import uk.ac.ebi.eva.commons.batch.job.JobStatusManager; +import uk.ac.ebi.eva.remapping.source.parameters.InputParameters; import java.util.Collection; @Component -public class AccessionRemappingJobLauncherCommandLineRunner extends JobLauncherCommandLineRunner implements +public class AccessionRemappingJobLauncherCommandLineRunner extends JobLauncherApplicationRunner implements ApplicationEventPublisherAware, ExitCodeGenerator { private static final Logger logger = LoggerFactory.getLogger(AccessionRemappingJobLauncherCommandLineRunner.class); @@ -92,9 +90,9 @@ public void setJobs(Collection jobs) { } @Override - public void setJobNames(String jobName) { + public void setJobName(String jobName) { this.jobName = jobName; - super.setJobNames(jobName); + super.setJobName(jobName); } @Override @@ -119,17 +117,16 @@ public void run(String... args) throws JobExecutionException { if (inputParameters.isForceRestart()) { JobExecution previousJobExecution = CommandLineRunnerUtils.getLastJobExecution(jobName, jobExplorer, - jobParameters); + jobParameters); CommandLineRunnerUtils.markPreviousJobAsFailed(jobName, jobRepository, previousJobExecution != null ? previousJobExecution.getJobParameters() : jobParameters); - } - else { + } else { jobParameters = CommandLineRunnerUtils.addRunIDToJobParameters(jobName, jobExplorer, jobParameters); } launchJob(jobParameters); } catch (NoJobToExecuteException | NoParametersHaveBeenPassedException | NoPreviousJobExecutionException - | UnknownJobException | JobParametersInvalidException | JobExecutionAlreadyRunningException e) { + | UnknownJobException | JobParametersInvalidException | JobExecutionAlreadyRunningException e) { logger.error(e.getMessage()); logger.debug("Error trace", e); abnormalExit = true; @@ -151,8 +148,7 @@ private void launchJob(JobParameters jobParameters) throws JobExecutionException @Override protected void execute(Job job, JobParameters jobParameters) throws JobExecutionAlreadyRunningException, - JobRestartException, JobInstanceAlreadyCompleteException, JobParametersInvalidException, - JobParametersNotFoundException { + JobRestartException, JobInstanceAlreadyCompleteException, JobParametersInvalidException { logger.info("Running job '" + jobName + "' with parameters: " + jobParameters); super.execute(job, jobParameters); } diff --git a/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/batch/io/DbsnpSubmittedVariantMongoReaderTest.java b/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/batch/io/DbsnpSubmittedVariantMongoReaderTest.java index 83292677e..c397c49ab 100644 --- a/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/batch/io/DbsnpSubmittedVariantMongoReaderTest.java +++ b/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/batch/io/DbsnpSubmittedVariantMongoReaderTest.java @@ -16,26 +16,22 @@ package uk.ac.ebi.eva.remapping.source.batch.io; -import com.lordofthejars.nosqlunit.annotation.UsingDataSet; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.batch.item.ExecutionContext; import org.springframework.batch.item.ItemReader; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.ApplicationContext; +import org.springframework.core.io.ResourceLoader; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; - +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.MongoConfiguration; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpSubmittedVariantEntity; -import uk.ac.ebi.eva.accession.core.test.rule.FixSpringMongoDbRule; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; +import uk.ac.ebi.eva.accession.core.utils.MongoTestDataLoader; import uk.ac.ebi.eva.remapping.source.test.configuration.MongoTestConfiguration; import java.util.ArrayList; @@ -43,43 +39,38 @@ import java.util.Collections; import java.util.List; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; -@RunWith(SpringRunner.class) +@ExtendWith(SpringExtension.class) @TestPropertySource("classpath:application.properties") -@UsingDataSet(locations = { - "/test-data/dbsnpSubmittedVariantEntity.json" -}) @ContextConfiguration(classes = {MongoConfiguration.class, MongoTestConfiguration.class}) -public class DbsnpSubmittedVariantMongoReaderTest { - - private static final String TEST_DB = "test-db"; +public class DbsnpSubmittedVariantMongoReaderTest extends MongoTestContainerHelper { private static final String ASSEMBLY = "GCA_000409795.2"; @Autowired private MongoTemplate mongoTemplate; - //Required by nosql-unit @Autowired - private ApplicationContext applicationContext; - - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName(TEST_DB).build()); + private ResourceLoader resourceLoader; private DbsnpSubmittedVariantMongoReader reader; - @Before + @BeforeEach public void setUp() { + mongoTemplate.getDb().drop(); + + new MongoTestDataLoader(mongoTemplate, resourceLoader).load("/test-data/dbsnpSubmittedVariantEntity.json"); + ExecutionContext executionContext = new ExecutionContext(); reader = new DbsnpSubmittedVariantMongoReader(ASSEMBLY, mongoTemplate, null, 0); reader.open(executionContext); } - @After + @AfterEach public void tearDown() { reader.close(); + mongoTemplate.getDb().drop(); } @Test diff --git a/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/batch/io/EvaSubmittedVariantMongoReaderTest.java b/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/batch/io/EvaSubmittedVariantMongoReaderTest.java index 44c1005e1..6925f93e2 100644 --- a/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/batch/io/EvaSubmittedVariantMongoReaderTest.java +++ b/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/batch/io/EvaSubmittedVariantMongoReaderTest.java @@ -16,69 +16,59 @@ package uk.ac.ebi.eva.remapping.source.batch.io; -import com.lordofthejars.nosqlunit.annotation.UsingDataSet; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.batch.item.ExecutionContext; import org.springframework.batch.item.ItemReader; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.ApplicationContext; +import org.springframework.core.io.ResourceLoader; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; - +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.MongoConfiguration; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; -import uk.ac.ebi.eva.accession.core.test.rule.FixSpringMongoDbRule; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; +import uk.ac.ebi.eva.accession.core.utils.MongoTestDataLoader; import uk.ac.ebi.eva.remapping.source.test.configuration.MongoTestConfiguration; import java.util.ArrayList; import java.util.Collections; import java.util.List; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; -@RunWith(SpringRunner.class) +@ExtendWith(SpringExtension.class) @TestPropertySource("classpath:application.properties") -@UsingDataSet(locations = { - "/test-data/submittedVariantEntity.json" -}) @ContextConfiguration(classes = {MongoConfiguration.class, MongoTestConfiguration.class}) -public class EvaSubmittedVariantMongoReaderTest { - - private static final String TEST_DB = "test-db"; - +public class EvaSubmittedVariantMongoReaderTest extends MongoTestContainerHelper { private static final String ASSEMBLY = "GCA_000409795.2"; @Autowired private MongoTemplate mongoTemplate; - //Required by nosql-unit @Autowired - private ApplicationContext applicationContext; - - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName(TEST_DB).build()); + private ResourceLoader resourceLoader; private EvaSubmittedVariantMongoReader reader; - @Before + @BeforeEach public void setUp() { + mongoTemplate.getDb().drop(); + + new MongoTestDataLoader(mongoTemplate, resourceLoader).load("/test-data/submittedVariantEntity.json"); + ExecutionContext executionContext = new ExecutionContext(); reader = new EvaSubmittedVariantMongoReader(ASSEMBLY, mongoTemplate, null, 0); reader.open(executionContext); } - @After + @AfterEach public void tearDown() { reader.close(); + mongoTemplate.getDb().drop(); } @Test diff --git a/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/batch/io/VariantContextWriterTest.java b/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/batch/io/VariantContextWriterTest.java index 97ce1cfd3..d1294bb82 100644 --- a/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/batch/io/VariantContextWriterTest.java +++ b/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/batch/io/VariantContextWriterTest.java @@ -18,11 +18,10 @@ package uk.ac.ebi.eva.remapping.source.batch.io; import htsjdk.variant.variantcontext.VariantContext; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; - +import org.junit.jupiter.api.Test; +import org.springframework.batch.item.Chunk; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; +import uk.ac.ebi.eva.accession.core.utils.PipelineTemporaryFolderUtil; import uk.ac.ebi.eva.remapping.source.batch.processors.SubmittedVariantToVariantContextProcessor; import uk.ac.ebi.eva.remapping.source.parameters.ReportPathResolver; @@ -37,8 +36,8 @@ import java.util.stream.Collectors; import java.util.stream.Stream; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; public class VariantContextWriterTest { @@ -64,12 +63,11 @@ public class VariantContextWriterTest { public static final String SS_HASH = "hash1"; - @Rule - public TemporaryFolder temporaryFolder = new TemporaryFolder(); + public PipelineTemporaryFolderUtil temporaryFolderUtil = new PipelineTemporaryFolderUtil(); @Test public void basicWrite() throws Exception { - File outputFolder = temporaryFolder.newFolder(); + File outputFolder = temporaryFolderUtil.newFolder(); assertWriteVcf(outputFolder, buildVariant("1", 1000, "C", "A", 123L)); } @@ -84,7 +82,7 @@ private File assertWriteVcf(File outputFolder, SubmittedVariantEntity... variant List variantContexts = Stream.of(variants).map(variantToVariantContextProcessor::process).collect(Collectors.toList()); - writer.write(variantContexts); + writer.write(new Chunk<>(variantContexts)); writer.close(); @@ -106,9 +104,9 @@ private SubmittedVariantEntity buildVariant(String chr, long start, String ref, private SubmittedVariantEntity buildVariant(String chr, long start, String ref, String alt, Long rs, Long backpropRs, String project, int taxonomy) { SubmittedVariantEntity submittedVariantEntity = new SubmittedVariantEntity(1L, SS_HASH, REFERENCE_ASSEMBLY, - taxonomy, project, chr, start, ref, - alt, rs, false, false, false, false, - 1); + taxonomy, project, chr, start, ref, + alt, rs, false, false, false, false, + 1); if (backpropRs != null) { submittedVariantEntity.setBackPropagatedVariantAccession(backpropRs); } @@ -118,7 +116,7 @@ private SubmittedVariantEntity buildVariant(String chr, long start, String ref, @Test public void writeBasicFields() throws Exception { - File outputFolder = temporaryFolder.newFolder(); + File outputFolder = temporaryFolderUtil.newFolder(); String chromosome = "1"; long start = 1000; String reference = "C"; @@ -151,7 +149,7 @@ private long forEachVcfDataLine(File output, Consumer processVcfDataLi @Test public void writeRsId() throws Exception { - File outputFolder = temporaryFolder.newFolder(); + File outputFolder = temporaryFolderUtil.newFolder(); long rsId = 123L; File output = assertWriteVcf(outputFolder, buildVariant("1", 1000, "C", "A", rsId)); @@ -186,11 +184,11 @@ private void assertInfo(String infoColumn, String expectedSSHash, Long expectedR assertEquals("RS=rs" + expectedRsId, info); } else if (info.startsWith("BACKPROP_RS=")) { assertEquals("BACKPROP_RS=rs" + expectedBackpropRs, info); - } else if(info.startsWith("PROJECT=")) { + } else if (info.startsWith("PROJECT=")) { assertEquals("PROJECT=" + expectedProject, info); - } else if(info.startsWith("TAX=")) { + } else if (info.startsWith("TAX=")) { assertEquals("TAX=" + expectedTaxonomy, info); - } else if(info.startsWith("CREATED=")) { + } else if (info.startsWith("CREATED=")) { assertEquals("CREATED=" + expectedCreatedDate, info); } } @@ -198,7 +196,7 @@ private void assertInfo(String infoColumn, String expectedSSHash, Long expectedR @Test public void writeProjectAndTaxonomy() throws Exception { - File outputFolder = temporaryFolder.newFolder(); + File outputFolder = temporaryFolderUtil.newFolder(); String project2 = "project2"; int taxonomy2 = 1000; File output = assertWriteVcf(outputFolder, buildVariant("1", 1000, "C", "A", null, project2, taxonomy2)); @@ -212,7 +210,7 @@ public void writeProjectAndTaxonomy() throws Exception { @Test public void writeProjectAndTaxonomyAndRs() throws Exception { - File outputFolder = temporaryFolder.newFolder(); + File outputFolder = temporaryFolderUtil.newFolder(); String project2 = "project2"; int taxonomy2 = 1000; long rsId = 123L; @@ -228,16 +226,16 @@ public void writeProjectAndTaxonomyAndRs() throws Exception { @Test public void writeSeveralVariants() throws Exception { - File outputFolder = temporaryFolder.newFolder(); + File outputFolder = temporaryFolderUtil.newFolder(); String project2 = "project2"; int taxonomy2 = 1000; long rsId = 123L; long backpropRs = 456L; File output = assertWriteVcf(outputFolder, - buildVariant("1", 1000, "C", "A", rsId, backpropRs, project2, taxonomy2), - buildVariant("2", 1000, "C", "A", rsId, backpropRs, project2, taxonomy2), - buildVariant("3", 1000, "C", "A", rsId, backpropRs, project2, taxonomy2), - buildVariant("4", 1000, "C", "A", rsId, backpropRs, project2, taxonomy2)); + buildVariant("1", 1000, "C", "A", rsId, backpropRs, project2, taxonomy2), + buildVariant("2", 1000, "C", "A", rsId, backpropRs, project2, taxonomy2), + buildVariant("3", 1000, "C", "A", rsId, backpropRs, project2, taxonomy2), + buildVariant("4", 1000, "C", "A", rsId, backpropRs, project2, taxonomy2)); final Long[] expectedChr = {1L}; long variantCount = forEachVcfDataLine(output, (String[] columns) -> { @@ -251,10 +249,10 @@ public void writeSeveralVariants() throws Exception { @Test public void writeProjectWithSpecialCharacters() throws Exception { - File outputFolder = temporaryFolder.newFolder(); + File outputFolder = temporaryFolderUtil.newFolder(); String specialProject = "a%weird=project;with,special characters"; File output = assertWriteVcf(outputFolder, buildVariant("1", 1000, "C", "A", null, specialProject, - TAXONOMY_ACCESSION)); + TAXONOMY_ACCESSION)); long variantCount = forEachVcfDataLine(output, (String[] columns) -> { assertEquals(COLUMNS_IN_VCF_WITHOUT_SAMPLES, columns.length); diff --git a/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/batch/processors/ContextNucleotideAdditionProcessorTest.java b/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/batch/processors/ContextNucleotideAdditionProcessorTest.java index 06a764df5..96623a5a7 100644 --- a/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/batch/processors/ContextNucleotideAdditionProcessorTest.java +++ b/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/batch/processors/ContextNucleotideAdditionProcessorTest.java @@ -17,10 +17,9 @@ */ package uk.ac.ebi.eva.remapping.source.batch.processors; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; - +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; import uk.ac.ebi.eva.accession.core.batch.io.FastaSynonymSequenceReader; import uk.ac.ebi.eva.accession.core.contig.ContigMapping; import uk.ac.ebi.eva.accession.core.contig.ContigSynonyms; @@ -31,9 +30,10 @@ import java.nio.file.Paths; import java.util.Arrays; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThrows; -import static org.junit.Assert.fail; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; public class ContextNucleotideAdditionProcessorTest { @@ -51,18 +51,18 @@ public class ContextNucleotideAdditionProcessorTest { private static ContextNucleotideAdditionProcessor contextNucleotideAdditionProcessor; - @BeforeClass + @BeforeAll public static void setUpClass() throws Exception { Path fastaPath = Paths.get("../eva-accession-core/src/test/resources/input-files/fasta/Gallus_gallus-5.0.test.fa"); ContigMapping contigMapping = new ContigMapping( Arrays.asList(new ContigSynonyms(CONTIG, "", "", "", "", "", true), - new ContigSynonyms(SCAFFOLD, "", "", SCAFFOLD_GENBANK_IN_FASTA, "", "", true), - new ContigSynonyms(MISSING_IN_FASTA, "", "", "", "", "", true))); + new ContigSynonyms(SCAFFOLD, "", "", SCAFFOLD_GENBANK_IN_FASTA, "", "", true), + new ContigSynonyms(MISSING_IN_FASTA, "", "", "", "", "", true))); fastaSynonymSequenceReader = new FastaSynonymSequenceReader(contigMapping, fastaPath); contextNucleotideAdditionProcessor = new ContextNucleotideAdditionProcessor(fastaSynonymSequenceReader); } - @AfterClass + @AfterAll public static void tearDownClass() throws Exception { fastaSynonymSequenceReader.close(); } @@ -97,8 +97,9 @@ public void testNonEmptyAlleles() throws Exception { private SubmittedVariantEntity createVariant(String chromosome, long start, String reference, String alternate) { return new SubmittedVariantEntity(1L, "hash", "GCA_x", 9999, "project", chromosome, start, reference, alternate, - 100L, false, false, false, false, 1); + 100L, false, false, false, false, 1); } + @Test public void testINDELStartPos1() throws Exception { SubmittedVariantEntity variant = createVariant(CONTIG, 1, "", "A"); @@ -179,15 +180,18 @@ public void addContextBaseUsingSynonymContig() throws Exception { assertEquals("TA", processedVariant.getAlternateAllele()); } - @Test(expected = IllegalArgumentException.class) - public void contigNotFound() throws Exception { + @Test + public void contigNotFound() { SubmittedVariantEntity variant1 = createVariant(MISSING_IN_FASTA, 10, "", "A"); - try { - contextNucleotideAdditionProcessor.process(variant1); - } catch (PositionOutsideOfContigException wrongException) { - fail("The exception (" + wrongException.getClass().getSimpleName() - + ") is wrong because the variant doesn't have a position outside of chromosome. The correct " - + "exception should be that the contig is not present in the fasta"); + + Exception exception = assertThrows(Exception.class, () -> contextNucleotideAdditionProcessor.process(variant1)); + + if (exception instanceof PositionOutsideOfContigException) { + fail("The exception (" + exception.getClass().getSimpleName() + + ") is wrong because the variant doesn't have a position outside of chromosome. The correct " + + "exception should be that the contig is not present in the fasta"); } + + assertTrue(exception instanceof IllegalArgumentException); } } diff --git a/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/batch/processors/ExcludeAllelesMismatchVariantsProcessorTest.java b/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/batch/processors/ExcludeAllelesMismatchVariantsProcessorTest.java index 7ffd57ff1..9f23962b0 100644 --- a/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/batch/processors/ExcludeAllelesMismatchVariantsProcessorTest.java +++ b/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/batch/processors/ExcludeAllelesMismatchVariantsProcessorTest.java @@ -15,8 +15,8 @@ */ package uk.ac.ebi.eva.remapping.source.batch.processors; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import uk.ac.ebi.ampt2d.commons.accession.hashing.SHA1HashingFunction; import uk.ac.ebi.eva.accession.core.model.ISubmittedVariant; @@ -26,8 +26,8 @@ import java.util.function.Function; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; public class ExcludeAllelesMismatchVariantsProcessorTest { @@ -36,7 +36,7 @@ public class ExcludeAllelesMismatchVariantsProcessorTest { private ExcludeAllelesMismatchVariantsProcessor processor; - @Before + @BeforeEach public void setUp() { processor = new ExcludeAllelesMismatchVariantsProcessor(); } @@ -59,6 +59,6 @@ private SubmittedVariantEntity createSubmittedVariantEntity(Boolean allelesMatch SubmittedVariant submittedVariant = new SubmittedVariant("asm", 1000, "project", "contig", 100, "A", "T", null); String hash = hashingFunction.apply(submittedVariant); return new SubmittedVariantEntity(5000000000L, hash, "asm", 1000, "project", "contig", 100, "A", "T", null, - false, false, allelesMatch, false, 1, null); + false, false, allelesMatch, false, 1, null); } } diff --git a/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/batch/processors/ExcludeMultimapVariantsProcessorTest.java b/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/batch/processors/ExcludeMultimapVariantsProcessorTest.java index 579c4a41e..4bfa19530 100644 --- a/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/batch/processors/ExcludeMultimapVariantsProcessorTest.java +++ b/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/batch/processors/ExcludeMultimapVariantsProcessorTest.java @@ -15,8 +15,8 @@ */ package uk.ac.ebi.eva.remapping.source.batch.processors; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import uk.ac.ebi.ampt2d.commons.accession.hashing.SHA1HashingFunction; import uk.ac.ebi.eva.accession.core.model.ISubmittedVariant; @@ -26,8 +26,8 @@ import java.util.function.Function; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; public class ExcludeMultimapVariantsProcessorTest { @@ -35,7 +35,7 @@ public class ExcludeMultimapVariantsProcessorTest { private ExcludeMultimapVariantsProcessor processor; - @Before + @BeforeEach public void setUp() { processor = new ExcludeMultimapVariantsProcessor(); submittedHashingFunction = new SubmittedVariantSummaryFunction().andThen(new SHA1HashingFunction()); @@ -66,9 +66,9 @@ private SubmittedVariantEntity createSubmittedVariantEntity(Integer mapWeight) { SubmittedVariant submittedVariant = new SubmittedVariant("asm", 1000, "project", "contig", 100, "A", "T", null); String hash = submittedHashingFunction.apply(submittedVariant); SubmittedVariantEntity submittedVariantEntity = new SubmittedVariantEntity(5000000000L, hash, "asm", 1000, - "project", "contig", 100, "A", "T", - null, false, false, false, false, 1, - mapWeight); + "project", "contig", 100, "A", "T", + null, false, false, false, false, 1, + mapWeight); return submittedVariantEntity; } } \ No newline at end of file diff --git a/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/batch/processors/VariantToVariantContextProcessorTest.java b/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/batch/processors/VariantToVariantContextProcessorTest.java index b9d2d4095..413936119 100644 --- a/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/batch/processors/VariantToVariantContextProcessorTest.java +++ b/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/batch/processors/VariantToVariantContextProcessorTest.java @@ -18,17 +18,16 @@ import htsjdk.variant.variantcontext.Allele; import htsjdk.variant.variantcontext.VariantContext; -import org.junit.Before; -import org.junit.Test; - +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; import java.time.LocalDateTime; import java.util.Collections; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThrows; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; public class VariantToVariantContextProcessorTest { @@ -50,7 +49,7 @@ public class VariantToVariantContextProcessorTest { private SubmittedVariantToVariantContextProcessor variantConverter; - @Before + @BeforeEach public void setUp() { variantConverter = new SubmittedVariantToVariantContextProcessor(); } @@ -65,8 +64,8 @@ public void SNV() { private SubmittedVariantEntity buildVariant(String chr1, int start, String reference, String alternate, String project, int taxonomy) { SubmittedVariantEntity variant = new SubmittedVariantEntity(SS_ID, "hash", "assembly", taxonomy, project, chr1, - start, reference, alternate, RS_ID, - false, false, false, false, 1); + start, reference, alternate, RS_ID, + false, false, false, false, 1); variant.setCreatedDate(LocalDateTime.parse(CREATED_DATE)); return variant; } @@ -78,7 +77,7 @@ private void assertVariantContext(VariantContext variantContext, Long expectedId assertEquals(expectedStart, variantContext.getStart()); assertEquals(Allele.create(expectedReference, true), variantContext.getReference()); assertEquals(Collections.singletonList(Allele.create(expectedAlternate, false)), - variantContext.getAlternateAlleles()); + variantContext.getAlternateAlleles()); assertEquals("ss" + expectedId, variantContext.getID()); assertTrue(variantContext.getFilters().isEmpty()); assertEquals(5, variantContext.getCommonInfo().getAttributes().size()); @@ -88,7 +87,7 @@ private void assertVariantContext(VariantContext variantContext, Long expectedId assertTrue(variantContext.getCommonInfo().hasAttribute(TAXONOMY_ID_KEY)); assertEquals(expectedTaxonomy, - Integer.parseInt(variantContext.getCommonInfo().getAttribute(TAXONOMY_ID_KEY).toString())); + Integer.parseInt(variantContext.getCommonInfo().getAttribute(TAXONOMY_ID_KEY).toString())); assertEquals(0, variantContext.getSampleNames().size()); } diff --git a/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/configuration/batch/jobs/ExportSubmittedVariantsJobConfigurationTest.java b/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/configuration/batch/jobs/ExportSubmittedVariantsJobConfigurationTest.java index 6d2c0853b..6544bb0c9 100644 --- a/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/configuration/batch/jobs/ExportSubmittedVariantsJobConfigurationTest.java +++ b/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/configuration/batch/jobs/ExportSubmittedVariantsJobConfigurationTest.java @@ -15,31 +15,30 @@ */ package uk.ac.ebi.eva.remapping.source.configuration.batch.jobs; -import com.lordofthejars.nosqlunit.annotation.UsingDataSet; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.batch.core.BatchStatus; import org.springframework.batch.core.JobExecution; import org.springframework.batch.core.StepExecution; import org.springframework.batch.test.JobLauncherTestUtils; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.ApplicationContext; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.core.io.ResourceLoader; +import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; - -import uk.ac.ebi.eva.accession.core.test.rule.FixSpringMongoDbRule; +import org.springframework.test.context.junit.jupiter.SpringExtension; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; +import uk.ac.ebi.eva.accession.core.utils.MongoTestDataLoader; +import uk.ac.ebi.eva.commons.core.utils.FileUtils; +import uk.ac.ebi.eva.remapping.source.configuration.BeanNames; import uk.ac.ebi.eva.remapping.source.parameters.InputParameters; import uk.ac.ebi.eva.remapping.source.parameters.ReportPathResolver; +import uk.ac.ebi.eva.remapping.source.test.configuration.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.remapping.source.test.configuration.BatchTestConfiguration; import uk.ac.ebi.eva.remapping.source.test.configuration.MongoTestConfiguration; -import uk.ac.ebi.eva.commons.core.utils.FileUtils; -import uk.ac.ebi.eva.remapping.source.configuration.BeanNames; import java.io.FileInputStream; import java.io.FileNotFoundException; @@ -48,55 +47,56 @@ import java.util.List; import java.util.stream.Collectors; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static uk.ac.ebi.eva.remapping.source.test.configuration.BatchTestConfiguration.JOB_EXPORT_SUBMITTED_VARIANTS_JOB; -@RunWith(SpringRunner.class) -@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class}) -@UsingDataSet(locations = { - "/test-data/dbsnpSubmittedVariantEntity.json", - "/test-data/submittedVariantEntity.json"}) +@ExtendWith(SpringExtension.class) @TestPropertySource("classpath:with-projects.properties") -public class ExportSubmittedVariantsJobConfigurationTest { - - private static final String TEST_DB = "test-db"; - +@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class, + BatchJobRepositoryTestConfiguration.class}) +public class ExportSubmittedVariantsJobConfigurationTest extends MongoTestContainerHelper { private static final long EXPECTED_LINES_DBSNP = 3; private static final long EXPECTED_LINES_EVA = 1; @Autowired + @Qualifier(JOB_EXPORT_SUBMITTED_VARIANTS_JOB) private JobLauncherTestUtils jobLauncherTestUtils; @Autowired private InputParameters inputParameters; - //Required by nosql-unit @Autowired - private ApplicationContext applicationContext; + private MongoTemplate mongoTemplate; - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName(TEST_DB).build()); + @Autowired + private ResourceLoader resourceLoader; - @Before + @BeforeEach public void setUp() throws Exception { + mongoTemplate.getDb().drop(); deleteOutputFiles(); + + MongoTestDataLoader mongoTestDataLoader = new MongoTestDataLoader(mongoTemplate, resourceLoader); + mongoTestDataLoader.load("/test-data/dbsnpSubmittedVariantEntity.json"); + mongoTestDataLoader.load("/test-data/submittedVariantEntity.json"); } - @After + @AfterEach public void tearDown() throws Exception { + mongoTemplate.getDb().drop(); deleteOutputFiles(); } private void deleteOutputFiles() { ReportPathResolver.getDbsnpReportPath(inputParameters.getOutputFolder(), - inputParameters.getAssemblyAccession(), - inputParameters.getTaxonomy()) - .toFile().delete(); + inputParameters.getAssemblyAccession(), + inputParameters.getTaxonomy()) + .toFile().delete(); ReportPathResolver.getEvaReportPath(inputParameters.getOutputFolder(), - inputParameters.getAssemblyAccession(), - inputParameters.getTaxonomy()) - .toFile().delete(); + inputParameters.getAssemblyAccession(), + inputParameters.getTaxonomy()) + .toFile().delete(); } @Test @@ -104,7 +104,7 @@ public void basicJobCompletion() throws Exception { JobExecution jobExecution = jobLauncherTestUtils.launchJob(); List expectedSteps = Arrays.asList(BeanNames.EXPORT_EVA_SUBMITTED_VARIANTS_STEP, - BeanNames.EXPORT_DBSNP_SUBMITTED_VARIANTS_STEP); + BeanNames.EXPORT_DBSNP_SUBMITTED_VARIANTS_STEP); assertStepsExecuted(expectedSteps, jobExecution); assertEquals(BatchStatus.COMPLETED, jobExecution.getStatus()); @@ -122,16 +122,16 @@ public void variantsWritten() throws Exception { private FileInputStream getExportedEva() throws FileNotFoundException { return new FileInputStream(ReportPathResolver.getEvaReportPath(inputParameters.getOutputFolder(), - inputParameters.getAssemblyAccession(), - inputParameters.getTaxonomy()) - .toFile()); + inputParameters.getAssemblyAccession(), + inputParameters.getTaxonomy()) + .toFile()); } private FileInputStream getExportedDbsnp() throws FileNotFoundException { return new FileInputStream(ReportPathResolver.getDbsnpReportPath(inputParameters.getOutputFolder(), - inputParameters.getAssemblyAccession(), - inputParameters.getTaxonomy()) - .toFile()); + inputParameters.getAssemblyAccession(), + inputParameters.getTaxonomy()) + .toFile()); } private void assertStepsExecuted(List expectedSteps, JobExecution jobExecution) { diff --git a/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/runner/AccessionRemappingJobLauncherCommandLineRunnerTest.java b/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/runner/AccessionRemappingJobLauncherCommandLineRunnerTest.java index 8c61d6069..358595a01 100644 --- a/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/runner/AccessionRemappingJobLauncherCommandLineRunnerTest.java +++ b/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/runner/AccessionRemappingJobLauncherCommandLineRunnerTest.java @@ -15,72 +15,69 @@ */ package uk.ac.ebi.eva.remapping.source.runner; -import com.lordofthejars.nosqlunit.annotation.UsingDataSet; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.batch.core.JobExecutionException; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.ApplicationContext; +import org.springframework.core.io.ResourceLoader; +import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; - -import uk.ac.ebi.eva.accession.core.test.rule.FixSpringMongoDbRule; +import org.springframework.test.context.junit.jupiter.SpringExtension; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; +import uk.ac.ebi.eva.accession.core.utils.MongoTestDataLoader; import uk.ac.ebi.eva.remapping.source.parameters.InputParameters; import uk.ac.ebi.eva.remapping.source.parameters.ReportPathResolver; +import uk.ac.ebi.eva.remapping.source.test.configuration.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.remapping.source.test.configuration.BatchTestConfiguration; import uk.ac.ebi.eva.remapping.source.test.configuration.MongoTestConfiguration; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; -@RunWith(SpringRunner.class) -@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class}) -@UsingDataSet(locations = { - "/test-data/dbsnpSubmittedVariantEntity.json", - "/test-data/submittedVariantEntity.json"}) +@ExtendWith(SpringExtension.class) +@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class, + BatchJobRepositoryTestConfiguration.class}) @TestPropertySource("classpath:application.properties") -public class AccessionRemappingJobLauncherCommandLineRunnerTest { - - private static final String TEST_DB = "test-db"; - +public class AccessionRemappingJobLauncherCommandLineRunnerTest extends MongoTestContainerHelper { @Autowired private InputParameters inputParameters; @Autowired - private AccessionRemappingJobLauncherCommandLineRunner runner; + private MongoTemplate mongoTemplate; - //Required by nosql-unit @Autowired - private ApplicationContext applicationContext; + private ResourceLoader resourceLoader; - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName(TEST_DB).build()); + @Autowired + private AccessionRemappingJobLauncherCommandLineRunner runner; - @Before + @BeforeEach public void setUp() { + mongoTemplate.getDb().drop(); deleteOutputFiles(); + + MongoTestDataLoader mongoTestDataLoader = new MongoTestDataLoader(mongoTemplate, resourceLoader); + mongoTestDataLoader.load("/test-data/dbsnpSubmittedVariantEntity.json"); + mongoTestDataLoader.load("/test-data/submittedVariantEntity.json"); } - @After + @AfterEach public void tearDown() { + mongoTemplate.getDb().drop(); deleteOutputFiles(); } private void deleteOutputFiles() { ReportPathResolver.getDbsnpReportPath(inputParameters.getOutputFolder(), - inputParameters.getAssemblyAccession(), - inputParameters.getTaxonomy()) - .toFile().delete(); + inputParameters.getAssemblyAccession(), + inputParameters.getTaxonomy()) + .toFile().delete(); ReportPathResolver.getEvaReportPath(inputParameters.getOutputFolder(), - inputParameters.getAssemblyAccession(), - inputParameters.getTaxonomy()) - .toFile().delete(); + inputParameters.getAssemblyAccession(), + inputParameters.getTaxonomy()) + .toFile().delete(); } @Test diff --git a/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/runner/AccessionRemappingJobLauncherCommandLineRunnerWithProjectsTest.java b/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/runner/AccessionRemappingJobLauncherCommandLineRunnerWithProjectsTest.java index f246f14df..37a40d69a 100644 --- a/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/runner/AccessionRemappingJobLauncherCommandLineRunnerWithProjectsTest.java +++ b/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/runner/AccessionRemappingJobLauncherCommandLineRunnerWithProjectsTest.java @@ -15,38 +15,32 @@ */ package uk.ac.ebi.eva.remapping.source.runner; -import com.lordofthejars.nosqlunit.annotation.UsingDataSet; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.batch.core.JobExecutionException; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.ApplicationContext; +import org.springframework.core.io.ResourceLoader; +import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; - -import uk.ac.ebi.eva.accession.core.test.rule.FixSpringMongoDbRule; +import org.springframework.test.context.junit.jupiter.SpringExtension; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; +import uk.ac.ebi.eva.accession.core.utils.MongoTestDataLoader; import uk.ac.ebi.eva.remapping.source.parameters.InputParameters; import uk.ac.ebi.eva.remapping.source.parameters.ReportPathResolver; +import uk.ac.ebi.eva.remapping.source.test.configuration.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.remapping.source.test.configuration.BatchTestConfiguration; import uk.ac.ebi.eva.remapping.source.test.configuration.MongoTestConfiguration; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; -@RunWith(SpringRunner.class) -@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class}) -@UsingDataSet(locations = { - "/test-data/dbsnpSubmittedVariantEntity.json", - "/test-data/submittedVariantEntity.json"}) +@ExtendWith(SpringExtension.class) +@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class, + BatchJobRepositoryTestConfiguration.class}) @TestPropertySource("classpath:with-projects.properties") -public class AccessionRemappingJobLauncherCommandLineRunnerWithProjectsTest { - - private static final String TEST_DB = "test-db"; +public class AccessionRemappingJobLauncherCommandLineRunnerWithProjectsTest extends MongoTestContainerHelper { @Autowired private InputParameters inputParameters; @@ -54,33 +48,37 @@ public class AccessionRemappingJobLauncherCommandLineRunnerWithProjectsTest { @Autowired private AccessionRemappingJobLauncherCommandLineRunner runner; - //Required by nosql-unit @Autowired - private ApplicationContext applicationContext; + private MongoTemplate mongoTemplate; - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName(TEST_DB).build()); + @Autowired + private ResourceLoader resourceLoader; - @Before + @BeforeEach public void setUp() { + mongoTemplate.getDb().drop(); deleteOutputFiles(); + + MongoTestDataLoader mongoTestDataLoader = new MongoTestDataLoader(mongoTemplate, resourceLoader); + mongoTestDataLoader.load("/test-data/dbsnpSubmittedVariantEntity.json"); + mongoTestDataLoader.load("/test-data/submittedVariantEntity.json"); } - @After + @AfterEach public void tearDown() { deleteOutputFiles(); + mongoTemplate.getDb().drop(); } private void deleteOutputFiles() { ReportPathResolver.getDbsnpReportPath(inputParameters.getOutputFolder(), - inputParameters.getAssemblyAccession(), - inputParameters.getTaxonomy()) - .toFile().delete(); + inputParameters.getAssemblyAccession(), + inputParameters.getTaxonomy()) + .toFile().delete(); ReportPathResolver.getEvaReportPath(inputParameters.getOutputFolder(), - inputParameters.getAssemblyAccession(), - inputParameters.getTaxonomy()) - .toFile().delete(); + inputParameters.getAssemblyAccession(), + inputParameters.getTaxonomy()) + .toFile().delete(); } @Test diff --git a/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/test/configuration/BatchJobRepositoryTestConfiguration.java b/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/test/configuration/BatchJobRepositoryTestConfiguration.java new file mode 100644 index 000000000..57c837cc4 --- /dev/null +++ b/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/test/configuration/BatchJobRepositoryTestConfiguration.java @@ -0,0 +1,22 @@ +package uk.ac.ebi.eva.remapping.source.test.configuration; + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Primary; +import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseBuilder; +import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseType; + +import javax.sql.DataSource; + +@Configuration +public class BatchJobRepositoryTestConfiguration { + + @Bean + @Primary + public DataSource dataSource() { + return new EmbeddedDatabaseBuilder() + .setType(EmbeddedDatabaseType.HSQL) + .addScript("org/springframework/batch/core/schema-hsqldb.sql") + .build(); + } +} \ No newline at end of file diff --git a/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/test/configuration/BatchTestConfiguration.java b/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/test/configuration/BatchTestConfiguration.java index 8a79307b5..9a15f97c3 100644 --- a/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/test/configuration/BatchTestConfiguration.java +++ b/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/test/configuration/BatchTestConfiguration.java @@ -16,30 +16,42 @@ package uk.ac.ebi.eva.remapping.source.test.configuration; +import org.springframework.batch.core.Job; +import org.springframework.batch.core.launch.JobLauncher; +import org.springframework.batch.core.repository.JobRepository; import org.springframework.batch.test.JobLauncherTestUtils; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Import; - import uk.ac.ebi.eva.accession.core.configuration.nonhuman.MongoConfiguration; +import uk.ac.ebi.eva.commons.batch.job.JobExecutionApplicationListener; import uk.ac.ebi.eva.remapping.source.configuration.InputParametersConfiguration; import uk.ac.ebi.eva.remapping.source.configuration.batch.jobs.ExportSubmittedVariantsJobConfiguration; import uk.ac.ebi.eva.remapping.source.configuration.batch.policies.PoliciesConfiguration; import uk.ac.ebi.eva.remapping.source.runner.AccessionRemappingJobLauncherCommandLineRunner; -import uk.ac.ebi.eva.commons.batch.job.JobExecutionApplicationListener; + +import static uk.ac.ebi.eva.remapping.source.configuration.BeanNames.EXPORT_SUBMITTED_VARIANTS_JOB; @EnableAutoConfiguration @Import({MongoConfiguration.class, - InputParametersConfiguration.class, - ExportSubmittedVariantsJobConfiguration.class, - PoliciesConfiguration.class, - AccessionRemappingJobLauncherCommandLineRunner.class + InputParametersConfiguration.class, + ExportSubmittedVariantsJobConfiguration.class, + PoliciesConfiguration.class, + AccessionRemappingJobLauncherCommandLineRunner.class }) public class BatchTestConfiguration { - @Bean - public JobLauncherTestUtils jobLauncherTestUtils() { - return new JobLauncherTestUtils(); + public static final String JOB_EXPORT_SUBMITTED_VARIANTS_JOB = "JOB_EXPORT_SUBMITTED_VARIANTS_JOB"; + + @Bean(JOB_EXPORT_SUBMITTED_VARIANTS_JOB) + public JobLauncherTestUtils jobLauncherTestUtilsFromMongo(JobLauncher jobLauncher, JobRepository jobRepository, + @Qualifier(EXPORT_SUBMITTED_VARIANTS_JOB) Job job) { + JobLauncherTestUtils utils = new JobLauncherTestUtils(); + utils.setJobLauncher(jobLauncher); + utils.setJobRepository(jobRepository); + utils.setJob(job); + return utils; } @Bean diff --git a/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/test/configuration/MongoTestConfiguration.java b/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/test/configuration/MongoTestConfiguration.java index f8eed1fa0..1005a6f6a 100644 --- a/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/test/configuration/MongoTestConfiguration.java +++ b/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/test/configuration/MongoTestConfiguration.java @@ -17,12 +17,30 @@ */ package uk.ac.ebi.eva.remapping.source.test.configuration; +import com.mongodb.ReadPreference; +import com.mongodb.WriteConcern; +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoClients; +import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.domain.EntityScan; import org.springframework.boot.test.autoconfigure.data.mongo.AutoConfigureDataMongo; +import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Primary; +import org.springframework.data.mongodb.MongoDatabaseFactory; import org.springframework.data.mongodb.config.EnableMongoAuditing; +import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.data.mongodb.core.SimpleMongoClientDatabaseFactory; +import org.springframework.data.mongodb.core.WriteResultChecking; +import org.springframework.data.mongodb.core.convert.DefaultDbRefResolver; +import org.springframework.data.mongodb.core.convert.DefaultMongoTypeMapper; +import org.springframework.data.mongodb.core.convert.MappingMongoConverter; +import org.springframework.data.mongodb.core.convert.MongoCustomConversions; +import org.springframework.data.mongodb.core.mapping.MongoMappingContext; import org.springframework.data.mongodb.repository.config.EnableMongoRepositories; +import java.util.Collections; + @Configuration @EntityScan(basePackages = {"uk.ac.ebi.eva.accession.core.persistence"}) @EnableMongoRepositories(basePackages = { @@ -32,4 +50,56 @@ @AutoConfigureDataMongo public class MongoTestConfiguration { + @Value("${spring.data.mongodb.uri}") + private String mongoUri; + + @Value("${spring.data.mongodb.database}") + private String database; + + @Value("${mongodb.read-preference:primary}") + private String readPreference; + + @Bean + @Primary + public MongoClient mongoClient() { + return MongoClients.create(mongoUri); + } + + @Bean("primaryFactory") + @Primary + public MongoDatabaseFactory mongoDbFactory() { + return new SimpleMongoClientDatabaseFactory(mongoClient(), database); + } + + @Bean + @Primary + public MongoCustomConversions mongoCustomConversions() { + return new MongoCustomConversions(Collections.emptyList()); + } + + @Bean + @Primary + public MongoMappingContext mongoMappingContext(MongoCustomConversions conversions) { + MongoMappingContext context = new MongoMappingContext(); + context.setSimpleTypeHolder(conversions.getSimpleTypeHolder()); + context.setAutoIndexCreation(false); + return context; + } + + @Bean + @Primary + public MongoTemplate mongoTemplate(MongoMappingContext mongoMappingContext, + MongoCustomConversions conversions) { + MappingMongoConverter converter = new MappingMongoConverter( + new DefaultDbRefResolver(mongoDbFactory()), mongoMappingContext); + converter.setCustomConversions(conversions); + converter.setTypeMapper(new DefaultMongoTypeMapper(null)); + converter.afterPropertiesSet(); + MongoTemplate mongoTemplate = new MongoTemplate(mongoDbFactory(), converter); + mongoTemplate.setReadPreference(ReadPreference.valueOf(readPreference)); + mongoTemplate.setWriteConcern(WriteConcern.MAJORITY); + mongoTemplate.setWriteResultChecking(WriteResultChecking.EXCEPTION); + return mongoTemplate; + } + } diff --git a/eva-remapping-get-source/src/test/resources/application.properties b/eva-remapping-get-source/src/test/resources/application.properties index d8833dadf..941cf32a5 100644 --- a/eva-remapping-get-source/src/test/resources/application.properties +++ b/eva-remapping-get-source/src/test/resources/application.properties @@ -16,7 +16,6 @@ parameters.outputFolder=/tmp/ parameters.forceRestart=false parameters.chunkSize=1000 -spring.data.mongodb.uri=mongodb://|eva.mongo.host.test|:27017 spring.data.mongodb.database=test-db mongodb.read-preference=primary diff --git a/eva-remapping-get-source/src/test/resources/with-projects.properties b/eva-remapping-get-source/src/test/resources/with-projects.properties index fa9fd1511..d0c6bb68d 100644 --- a/eva-remapping-get-source/src/test/resources/with-projects.properties +++ b/eva-remapping-get-source/src/test/resources/with-projects.properties @@ -18,7 +18,6 @@ parameters.outputFolder=/tmp/ parameters.forceRestart=false parameters.chunkSize=1000 -spring.data.mongodb.uri=mongodb://|eva.mongo.host.test|:27017 spring.data.mongodb.database=test-db mongodb.read-preference=primary diff --git a/eva-remapping-ingest/pom.xml b/eva-remapping-ingest/pom.xml index 985e74985..d85d54e4b 100644 --- a/eva-remapping-ingest/pom.xml +++ b/eva-remapping-ingest/pom.xml @@ -2,12 +2,13 @@ + 4.0.0 + eva-accession uk.ac.ebi.eva - 0.6.58-SNAPSHOT + ${revision} - 4.0.0 eva-remapping-ingest jar @@ -27,6 +28,14 @@ xstream + + uk.ac.ebi.eva + eva-accession-core + ${revision} + test-jar + test + + org.springframework.batch spring-batch-test @@ -34,8 +43,18 @@ - com.lordofthejars - nosqlunit-mongodb + org.springframework.boot + spring-boot-testcontainers + test + + + org.testcontainers + mongodb + test + + + org.testcontainers + junit-jupiter test @@ -75,8 +94,4 @@ - - localhost - - \ No newline at end of file diff --git a/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/batch/io/RemappedSubmittedVariantsWriter.java b/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/batch/io/RemappedSubmittedVariantsWriter.java index e75f47636..a3232ee16 100644 --- a/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/batch/io/RemappedSubmittedVariantsWriter.java +++ b/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/batch/io/RemappedSubmittedVariantsWriter.java @@ -21,12 +21,12 @@ import org.apache.commons.lang3.tuple.ImmutableTriple; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.batch.item.Chunk; import org.springframework.batch.item.ItemWriter; -import org.springframework.dao.DuplicateKeyException; +import org.springframework.data.mongodb.BulkOperationException; import org.springframework.data.mongodb.core.BulkOperations; import org.springframework.data.mongodb.core.MongoTemplate; import uk.ac.ebi.ampt2d.commons.accession.core.models.EventType; - import uk.ac.ebi.eva.accession.core.exceptions.MongoBulkWriteExceptionUtils; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantInactiveEntity; @@ -75,9 +75,9 @@ public RemappedSubmittedVariantsWriter(MongoTemplate mongoTemplate, String assem } @Override - public void write(List submittedVariantsRemapped) { + public void write(Chunk submittedVariantsRemapped) { // We do this to avoid tedious generic type signatures in the method calls - List svesToInsert = new ArrayList<>(submittedVariantsRemapped); + List svesToInsert = new ArrayList<>(submittedVariantsRemapped.getItems()); // Resolve duplicate hashes and accessions before inserting; note that hash resolution must happen first. ImmutablePair, List> duplicateHashDiscards = @@ -86,26 +86,26 @@ public void write(List submittedVariantsRemapp resolveDuplicateAccessions(svesToInsert); List svesToDiscard = Stream.concat(duplicateHashDiscards.getLeft().stream(), - duplicateAccessionDiscards.getLeft().stream()) - .collect(Collectors.toList()); + duplicateAccessionDiscards.getLeft().stream()) + .collect(Collectors.toList()); List discardOperations = Stream.concat(duplicateHashDiscards.getRight().stream(), - duplicateAccessionDiscards.getRight().stream()) - .collect(Collectors.toList()); + duplicateAccessionDiscards.getRight().stream()) + .collect(Collectors.toList()); BulkOperations bulkOperations = mongoTemplate.bulkOps(BulkOperations.BulkMode.UNORDERED, - SubmittedVariantEntity.class, - collection); + SubmittedVariantEntity.class, + collection); // Deal with discards before inserts, to avoid DuplicateKeyExceptions if (discardOperations.size() > 0) { try { BulkOperations svoeBulkOperations = mongoTemplate.bulkOps(BulkOperations.BulkMode.UNORDERED, - SubmittedVariantOperationEntity.class, - operationCollection); + SubmittedVariantOperationEntity.class, + operationCollection); svoeBulkOperations.insert(discardOperations); BulkWriteResult bulkWriteResult = svoeBulkOperations.execute(); remappingIngestCounts.addRemappedVariantsDiscarded(bulkWriteResult.getInsertedCount()); - } catch (DuplicateKeyException exception) { + } catch (BulkOperationException exception) { // As we check for hash collisions in submitted variants, we should only get duplicate keys from // trying to insert an identical DISCARD operation, which shouldn't happen even when rerunning // (since identical SVEs are skipped). @@ -123,7 +123,7 @@ public void write(List submittedVariantsRemapp if (svesToDiscard.size() > 0) { bulkOperations.remove(query(where("_id").in( svesToDiscard.stream().map(SubmittedVariantEntity::getHashedMessage) - .collect(Collectors.toSet())))); + .collect(Collectors.toSet())))); bulkOperations.execute(); } @@ -138,17 +138,17 @@ private ImmutablePair, List svesToInsert) { Map> svesGroupedByHash = getDuplicateHashes(svesToInsert); return resolveDuplicates(svesToInsert, svesGroupedByHash, - "Submitted variant discarded due to duplicate hash"); + "Submitted variant discarded due to duplicate hash"); } private ImmutablePair, List> resolveDuplicateAccessions( List svesToInsert) { Map> svesGroupedByAccession = getDuplicateAccessions(svesToInsert); return resolveDuplicates(svesToInsert, svesGroupedByAccession, - "Submitted variant discarded due to duplicate SS IDs in assembly"); + "Submitted variant discarded due to duplicate SS IDs in assembly"); } - private ImmutablePair, List> resolveDuplicates( + private ImmutablePair, List> resolveDuplicates( List svesToInsert, Map> svesGroupedByKey, String discardMessage) { @@ -159,7 +159,7 @@ private ImmutablePair, List, LocalDateTime> createdDateMap = getAllCreatedDateFromSource(duplicateSves); - for (Map.Entry> keyAndSve: svesGroupedByKey.entrySet()) { + for (Map.Entry> keyAndSve : svesGroupedByKey.entrySet()) { List duplicates = keyAndSve.getValue(); // Ensure we only keep one out of the list of duplicates for this SS. SubmittedVariantEntity currentKept = duplicates.get(0); @@ -177,7 +177,7 @@ private ImmutablePair, ListImmutablePair, List> getDuplicateHashes(List svesToInsert) { List hashes = svesToInsert.stream() - .map(SubmittedVariantEntity::getHashedMessage) - .collect(Collectors.toList()); + .map(SubmittedVariantEntity::getHashedMessage) + .collect(Collectors.toList()); // Find duplicate hashes already in db List svesWithSameHash = mongoTemplate.find(query(where("_id").in(hashes)), - SubmittedVariantEntity.class, collection); + SubmittedVariantEntity.class, collection); // Get the subset of these that are actually identical (including accession) and remove them immediately from // svesToInsert. These are counted as skips rather than discards. Map> duplicateSve = filterForDuplicates( Stream.concat(svesToInsert.stream(), svesWithSameHash.stream()) - .collect(Collectors.groupingBy(sve -> sve.hashCode() + "_" + sve.getAccession()))); + .collect(Collectors.groupingBy(sve -> sve.hashCode() + "_" + sve.getAccession()))); for (List dups : duplicateSve.values()) { int numRemoved = removeAllFromSveList(svesToInsert, dups.get(0)); remappingIngestCounts.addRemappedVariantsSkipped(numRemoved); @@ -230,7 +230,7 @@ private Map> getDuplicateHashes(List> svesGroupedByHash = Stream.concat(svesToInsert.stream(), svesWithSameHash.stream()) - .collect(Collectors.groupingBy(SubmittedVariantEntity::getHashedMessage)); + .collect(Collectors.groupingBy(SubmittedVariantEntity::getHashedMessage)); return filterForDuplicates(svesGroupedByHash); } @@ -242,7 +242,7 @@ private int removeAllFromSveList(List sves, SubmittedVar } private boolean containsSveList(List sves, SubmittedVariantEntity sveToFind) { - for (SubmittedVariantEntity sve: sves) { + for (SubmittedVariantEntity sve : sves) { if (equalsSveAndAccession(sve, sveToFind)) { return true; } @@ -256,8 +256,8 @@ private boolean equalsSveAndAccession(SubmittedVariantEntity first, SubmittedVar private Map> getDuplicateAccessions(List svesToInsert) { List accessions = svesToInsert.stream() - .map(SubmittedVariantEntity::getAccession) - .collect(Collectors.toList()); + .map(SubmittedVariantEntity::getAccession) + .collect(Collectors.toList()); // Find duplicate SS already in the database List svesWithSameAccession = mongoTemplate.find( @@ -266,7 +266,7 @@ private Map> getDuplicateAccessions(List> svesGroupedBySs = Stream.concat(svesToInsert.stream(), svesWithSameAccession.stream()) - .collect(Collectors.groupingBy(SubmittedVariantEntity::getAccession)); + .collect(Collectors.groupingBy(SubmittedVariantEntity::getAccession)); return filterForDuplicates(svesGroupedBySs); } @@ -278,8 +278,8 @@ private Map, LocalDateTime> getAllCreatedD // duplicates in all cases. If there's a duplicate triple almost certainly the createdDates are the same. Map, LocalDateTime> targetToSourceCreatedDate = duplicateSves .stream().collect(Collectors.toMap(this::getKeyForCreatedDate, - SubmittedVariantEntity::getCreatedDate, - (cd1, cd2) -> Collections.min(Arrays.asList(cd1, cd2)))); + SubmittedVariantEntity::getCreatedDate, + (cd1, cd2) -> Collections.min(Arrays.asList(cd1, cd2)))); Map> svesBySourceAssembly = duplicateSves .stream().collect(Collectors.groupingBy( @@ -323,42 +323,42 @@ private LocalDateTime getCreatedDateFromSource(SubmittedVariantEntity targetSve, // If we can't find the source SVE, use the created date of the remapped SVE if (Objects.isNull(sourceSves) || sourceSves.isEmpty()) { logger.warn("No SS " + targetSve.getAccession() + " found in source assembly " - + targetSve.getRemappedFrom()); + + targetSve.getRemappedFrom()); return targetSve.getCreatedDate(); } // If we find multiple SS in the source, return the min created date if (sourceSves.size() > 1) { logger.warn("Duplicate SS " + targetSve.getAccession() + " found in source assembly " - + targetSve.getRemappedFrom()); + + targetSve.getRemappedFrom()); return Collections.min(sourceSves, Comparator.comparing(SubmittedVariantEntity::getCreatedDate)) - .getCreatedDate(); + .getCreatedDate(); } return sourceSves.get(0).getCreatedDate(); } private SubmittedVariantOperationEntity getDiscardOperationWithMessage(SubmittedVariantEntity sve, String message) { String svoeId = String.format("DISCARD_SS_%s_HASH_%s_SOURCE_%s", - sve.getAccession(), - sve.getHashedMessage(), - sve.getRemappedFrom()); + sve.getAccession(), + sve.getHashedMessage(), + sve.getRemappedFrom()); SubmittedVariantOperationEntity svoe = new SubmittedVariantOperationEntity(); svoe.fill(EventType.DISCARDED, - sve.getAccession(), - message, - Collections.singletonList(new SubmittedVariantInactiveEntity(sve))); + sve.getAccession(), + message, + Collections.singletonList(new SubmittedVariantInactiveEntity(sve))); svoe.setId(svoeId); return svoe; } - private Map> filterForDuplicates(Map> map) { + private Map> filterForDuplicates(Map> map) { return map.entrySet().stream() - .filter(candidateEntry -> candidateEntry.getValue().size() > 1) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + .filter(candidateEntry -> candidateEntry.getValue().size() > 1) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); } - private List flattenValues(Map> map) { + private List flattenValues(Map> map) { return map.values().stream() - .flatMap(List::stream) - .collect(Collectors.toList()); + .flatMap(List::stream) + .collect(Collectors.toList()); } } diff --git a/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/configuration/RunnerConfiguration.java b/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/configuration/RunnerConfiguration.java index f2e7e7111..1b4b4b90b 100644 --- a/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/configuration/RunnerConfiguration.java +++ b/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/configuration/RunnerConfiguration.java @@ -16,16 +16,10 @@ */ package uk.ac.ebi.eva.remapping.ingest.configuration; -import org.springframework.batch.core.configuration.annotation.BatchConfigurer; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; - -import uk.ac.ebi.eva.commons.batch.configuration.SpringBoot1CompatibilityConfiguration; import uk.ac.ebi.eva.commons.batch.job.JobExecutionApplicationListener; -import javax.persistence.EntityManagerFactory; -import javax.sql.DataSource; - @Configuration public class RunnerConfiguration { @@ -33,11 +27,4 @@ public class RunnerConfiguration { public JobExecutionApplicationListener jobExecutionApplicationListener() { return new JobExecutionApplicationListener(); } - - @Bean - public BatchConfigurer configurer(DataSource dataSource, EntityManagerFactory entityManagerFactory) - throws Exception { - return SpringBoot1CompatibilityConfiguration.getSpringBoot1CompatibleBatchConfigurer(dataSource, - entityManagerFactory); - } } diff --git a/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/io/VcfReaderConfiguration.java b/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/io/VcfReaderConfiguration.java index a725ed28c..d2fe6a889 100644 --- a/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/io/VcfReaderConfiguration.java +++ b/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/io/VcfReaderConfiguration.java @@ -20,15 +20,14 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Import; - import uk.ac.ebi.eva.commons.batch.io.AggregatedVcfLineMapper; import uk.ac.ebi.eva.commons.batch.io.UnwindingItemStreamReader; import uk.ac.ebi.eva.commons.batch.io.VcfReader; import uk.ac.ebi.eva.commons.core.models.Aggregation; import uk.ac.ebi.eva.commons.core.models.pipeline.Variant; +import uk.ac.ebi.eva.remapping.ingest.configuration.BeanNames; import uk.ac.ebi.eva.remapping.ingest.configuration.InputParametersConfiguration; import uk.ac.ebi.eva.remapping.ingest.parameters.InputParameters; -import uk.ac.ebi.eva.remapping.ingest.configuration.BeanNames; import java.io.File; import java.io.IOException; diff --git a/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/jobs/IngestRemappedVariantsFromVcfJobConfiguration.java b/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/jobs/IngestRemappedVariantsFromVcfJobConfiguration.java index 46684370f..d7cee50c2 100644 --- a/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/jobs/IngestRemappedVariantsFromVcfJobConfiguration.java +++ b/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/jobs/IngestRemappedVariantsFromVcfJobConfiguration.java @@ -18,8 +18,9 @@ import org.springframework.batch.core.Job; import org.springframework.batch.core.Step; import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; -import org.springframework.batch.core.configuration.annotation.JobBuilderFactory; +import org.springframework.batch.core.job.builder.JobBuilder; import org.springframework.batch.core.launch.support.RunIdIncrementer; +import org.springframework.batch.core.repository.JobRepository; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -36,12 +37,12 @@ public class IngestRemappedVariantsFromVcfJobConfiguration { public Job ingestRemappedVariantsFromVcfJob( @Qualifier(INGEST_REMAPPED_VARIANTS_FROM_VCF_STEP) Step ingestRemappedVariantsStep, @Qualifier(STORE_REMAPPING_METADATA_STEP) Step storeRemappingMetadataStep, - JobBuilderFactory jobBuilderFactory) { - return jobBuilderFactory.get(INGEST_REMAPPED_VARIANTS_FROM_VCF_JOB) - .incrementer(new RunIdIncrementer()) - .start(storeRemappingMetadataStep) - .next(ingestRemappedVariantsStep) - .build(); + JobRepository jobRepository) { + return new JobBuilder(INGEST_REMAPPED_VARIANTS_FROM_VCF_JOB, jobRepository) + .incrementer(new RunIdIncrementer()) + .start(storeRemappingMetadataStep) + .next(ingestRemappedVariantsStep) + .build(); } } diff --git a/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/steps/IngestRemappedFromVcfStepConfiguration.java b/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/steps/IngestRemappedFromVcfStepConfiguration.java index ba4973144..e123343ca 100644 --- a/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/steps/IngestRemappedFromVcfStepConfiguration.java +++ b/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/steps/IngestRemappedFromVcfStepConfiguration.java @@ -19,7 +19,8 @@ import org.springframework.batch.core.Step; import org.springframework.batch.core.StepExecutionListener; import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; -import org.springframework.batch.core.configuration.annotation.StepBuilderFactory; +import org.springframework.batch.core.repository.JobRepository; +import org.springframework.batch.core.step.builder.StepBuilder; import org.springframework.batch.core.step.tasklet.TaskletStep; import org.springframework.batch.item.ItemProcessor; import org.springframework.batch.item.ItemReader; @@ -28,7 +29,7 @@ import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; - +import org.springframework.transaction.PlatformTransactionManager; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; import uk.ac.ebi.eva.commons.core.models.IVariant; import uk.ac.ebi.eva.commons.core.models.pipeline.Variant; @@ -49,15 +50,15 @@ public Step ingestRemappedFromVcf( @Qualifier(COMPOSITE_VARIANT_PROCESSOR) ItemProcessor processor, @Qualifier(REMAPPED_SUBMITTED_VARIANTS_WRITER) ItemWriter submittedVariantWriter, @Qualifier(PROGRESS_LISTENER) StepExecutionListener progressListener, - StepBuilderFactory stepBuilderFactory, + JobRepository jobRepository, PlatformTransactionManager transactionManager, SimpleCompletionPolicy chunkSizeCompletionPolicy) { - TaskletStep step = stepBuilderFactory.get(BeanNames.INGEST_REMAPPED_VARIANTS_FROM_VCF_STEP) - .chunk(chunkSizeCompletionPolicy) - .reader(vcfReader) - .processor(processor) - .writer(submittedVariantWriter) - .listener(progressListener) - .build(); + TaskletStep step = new StepBuilder(BeanNames.INGEST_REMAPPED_VARIANTS_FROM_VCF_STEP, jobRepository) + .chunk(chunkSizeCompletionPolicy, transactionManager) + .reader(vcfReader) + .processor(processor) + .writer(submittedVariantWriter) + .listener(progressListener) + .build(); return step; } diff --git a/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/steps/StoreRemappingMetadataStepConfiguration.java b/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/steps/StoreRemappingMetadataStepConfiguration.java index d4b0a1a9b..6d2c6803b 100644 --- a/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/steps/StoreRemappingMetadataStepConfiguration.java +++ b/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/steps/StoreRemappingMetadataStepConfiguration.java @@ -18,11 +18,13 @@ import org.springframework.batch.core.Step; import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; -import org.springframework.batch.core.configuration.annotation.StepBuilderFactory; +import org.springframework.batch.core.repository.JobRepository; +import org.springframework.batch.core.step.builder.StepBuilder; import org.springframework.batch.core.step.tasklet.TaskletStep; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.transaction.PlatformTransactionManager; import uk.ac.ebi.eva.remapping.ingest.batch.tasklets.RemappingMetadata; import uk.ac.ebi.eva.remapping.ingest.batch.tasklets.StoreRemappingMetadataTasklet; @@ -38,20 +40,17 @@ public class StoreRemappingMetadataStepConfiguration { private final RemappingMetadata remappingMetadata; -// @Value("${build.version}") -// private String accessionVersion; - public StoreRemappingMetadataStepConfiguration(MongoTemplate mongoTemplate, RemappingMetadata remappingMetadata) { this.mongoTemplate = mongoTemplate; this.remappingMetadata = remappingMetadata; } @Bean(STORE_REMAPPING_METADATA_STEP) - public Step buildReportStep(StepBuilderFactory stepBuilderFactory) throws IOException { + public Step buildReportStep(JobRepository jobRepository, PlatformTransactionManager transactionManager) throws IOException { StoreRemappingMetadataTasklet tasklet = new StoreRemappingMetadataTasklet(mongoTemplate, remappingMetadata); - TaskletStep step = stepBuilderFactory.get(STORE_REMAPPING_METADATA_STEP) - .tasklet(tasklet) - .build(); + TaskletStep step = new StepBuilder(STORE_REMAPPING_METADATA_STEP, jobRepository) + .tasklet(tasklet, transactionManager) + .build(); return step; } } diff --git a/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/runner/IngestRemappedCommandLineRunner.java b/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/runner/IngestRemappedCommandLineRunner.java index c78936437..021002fd9 100644 --- a/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/runner/IngestRemappedCommandLineRunner.java +++ b/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/runner/IngestRemappedCommandLineRunner.java @@ -26,7 +26,6 @@ import org.springframework.batch.core.JobParametersInvalidException; import org.springframework.batch.core.explore.JobExplorer; import org.springframework.batch.core.launch.JobLauncher; -import org.springframework.batch.core.launch.JobParametersNotFoundException; import org.springframework.batch.core.repository.JobExecutionAlreadyRunningException; import org.springframework.batch.core.repository.JobInstanceAlreadyCompleteException; import org.springframework.batch.core.repository.JobRepository; @@ -34,11 +33,10 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.ExitCodeGenerator; -import org.springframework.boot.autoconfigure.batch.JobLauncherCommandLineRunner; +import org.springframework.boot.autoconfigure.batch.JobLauncherApplicationRunner; import org.springframework.context.ApplicationEventPublisherAware; import org.springframework.stereotype.Component; import org.springframework.util.PatternMatchUtils; - import uk.ac.ebi.eva.accession.core.runner.CommandLineRunnerUtils; import uk.ac.ebi.eva.commons.batch.exception.NoJobToExecuteException; import uk.ac.ebi.eva.commons.batch.exception.NoParametersHaveBeenPassedException; @@ -51,7 +49,7 @@ import java.util.Collection; @Component -public class IngestRemappedCommandLineRunner extends JobLauncherCommandLineRunner implements +public class IngestRemappedCommandLineRunner extends JobLauncherApplicationRunner implements ApplicationEventPublisherAware, ExitCodeGenerator { private static final Logger logger = LoggerFactory.getLogger(IngestRemappedCommandLineRunner.class); @@ -92,9 +90,9 @@ public void setJobs(Collection jobs) { } @Override - public void setJobNames(String jobName) { + public void setJobName(String jobName) { this.jobName = jobName; - super.setJobNames(jobName); + super.setJobName(jobName); } @Override @@ -118,16 +116,15 @@ public void run(String... args) throws JobExecutionException { JobStatusManager.checkIfPropertiesHaveBeenProvided(jobParameters); if (inputParameters.isForceRestart()) { JobExecution previousJobExecution = CommandLineRunnerUtils.getLastJobExecution(jobName, jobExplorer, - jobParameters); + jobParameters); CommandLineRunnerUtils.markPreviousJobAsFailed(jobName, jobRepository, previousJobExecution != null ? previousJobExecution.getJobParameters() : jobParameters); - } - else { + } else { jobParameters = CommandLineRunnerUtils.addRunIDToJobParameters(jobName, jobExplorer, jobParameters); } launchJob(jobParameters); } catch (NoJobToExecuteException | NoParametersHaveBeenPassedException | NoPreviousJobExecutionException - | UnknownJobException | JobParametersInvalidException | JobExecutionAlreadyRunningException e) { + | UnknownJobException | JobParametersInvalidException | JobExecutionAlreadyRunningException e) { logger.error(e.getMessage()); logger.debug("Error trace", e); abnormalExit = true; @@ -149,8 +146,7 @@ private void launchJob(JobParameters jobParameters) throws JobExecutionException @Override protected void execute(Job job, JobParameters jobParameters) throws JobExecutionAlreadyRunningException, - JobRestartException, JobInstanceAlreadyCompleteException, JobParametersInvalidException, - JobParametersNotFoundException { + JobRestartException, JobInstanceAlreadyCompleteException, JobParametersInvalidException { logger.info("Running job '" + jobName + "' with parameters: " + jobParameters); super.execute(job, jobParameters); } diff --git a/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/batch/io/RemappedSubmittedVariantsWriterTest.java b/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/batch/io/RemappedSubmittedVariantsWriterTest.java index 08f5e0e8e..316e09f93 100644 --- a/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/batch/io/RemappedSubmittedVariantsWriterTest.java +++ b/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/batch/io/RemappedSubmittedVariantsWriterTest.java @@ -16,30 +16,28 @@ */ package uk.ac.ebi.eva.remapping.ingest.batch.io; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; -import com.mongodb.MongoClient; -import org.junit.After; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.batch.item.Chunk; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.context.ApplicationContext; +import org.springframework.core.io.ResourceLoader; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.ampt2d.commons.accession.hashing.SHA1HashingFunction; - import uk.ac.ebi.eva.accession.core.model.ISubmittedVariant; import uk.ac.ebi.eva.accession.core.model.SubmittedVariant; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantOperationEntity; import uk.ac.ebi.eva.accession.core.summary.SubmittedVariantSummaryFunction; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.remapping.ingest.batch.listeners.RemappingIngestCounts; import uk.ac.ebi.eva.remapping.ingest.test.configuration.BatchTestConfiguration; -import uk.ac.ebi.eva.remapping.ingest.test.rule.FixSpringMongoDbRule; +import uk.ac.ebi.eva.remapping.ingest.test.configuration.MongoTestConfiguration; import java.time.LocalDateTime; import java.util.Arrays; @@ -49,21 +47,18 @@ import java.util.Set; import java.util.function.Function; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import static uk.ac.ebi.eva.remapping.ingest.configuration.BeanNames.REMAPPED_SUBMITTED_VARIANTS_WRITER; -@RunWith(SpringRunner.class) -@ContextConfiguration(classes = {BatchTestConfiguration.class}) +@ExtendWith(SpringExtension.class) +@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class}) @TestPropertySource("classpath:ingest-remapped-variants.properties") -public class RemappedSubmittedVariantsWriterTest { - - private static final String TEST_DB = "test-ingest-remapping"; - +public class RemappedSubmittedVariantsWriterTest extends MongoTestContainerHelper { @Autowired - private MongoClient mongoClient; + private MongoTemplate mongoTemplate; @Autowired - private MongoTemplate mongoTemplate; + private ResourceLoader resourceLoader; @Autowired @Qualifier(REMAPPED_SUBMITTED_VARIANTS_WRITER) @@ -72,27 +67,25 @@ public class RemappedSubmittedVariantsWriterTest { private final Function hashingFunction = new SubmittedVariantSummaryFunction().andThen(new SHA1HashingFunction()); - //Required by nosql-unit @Autowired - private ApplicationContext applicationContext; + private RemappingIngestCounts remappingIngestCounts; - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName(TEST_DB).build()); + @BeforeEach + public void setUp() { + mongoTemplate.getDb().drop(); + } - @Autowired - private RemappingIngestCounts remappingIngestCounts; - @After + @AfterEach public void tearDown() { - mongoClient.dropDatabase(TEST_DB); + mongoTemplate.getDb().drop(); remappingIngestCounts.resetCounts(); } private SubmittedVariantEntity createSve(Long accession, long start, String ref, String alt, LocalDateTime createdDate, String remappedFrom) { SubmittedVariant model = new SubmittedVariant("GCA_000000001.2", 1000, "projectId_1", "CM000002.1", - start, ref, alt, 3000000002L); + start, ref, alt, 3000000002L); String hash = hashingFunction.apply(model); SubmittedVariantEntity sve = new SubmittedVariantEntity(accession, hash, model, 1); sve.setCreatedDate(createdDate); @@ -104,7 +97,7 @@ private SubmittedVariantEntity createSveInAssembly(String assembly, Long accession, long start, String ref, String alt, LocalDateTime createdDate, String remappedFrom) { SubmittedVariant model = new SubmittedVariant(assembly, 1000, "projectId_1", "CM000002.1", - start, ref, alt, 3000000002L); + start, ref, alt, 3000000002L); String hash = hashingFunction.apply(model); SubmittedVariantEntity sve = new SubmittedVariantEntity(accession, hash, model, 1); sve.setCreatedDate(createdDate); @@ -137,7 +130,7 @@ public void testDoesNotWriteNewDuplicateAccessions_discardFromInput() { List svesToWrite = Collections.singletonList( // SVE remapped from newer source SVE should be discarded createSve(5000000002L, 2100, "C", "T", LocalDateTime.now(), "GCA_000000001.5")); - writer.write(svesToWrite); + writer.write(new Chunk<>(svesToWrite)); assertRemappingIngestCounts(0, 0, 1); assertDatabaseCounts(3); } @@ -150,7 +143,7 @@ public void testDoesNotWriteNewDuplicateAccessions_discardFromDatabase() { List svesToWrite = Collections.singletonList( createSve(5000000004L, 1100, "C", "T", LocalDateTime.now(), null)); - writer.write(svesToWrite); + writer.write(new Chunk<>(svesToWrite)); assertRemappingIngestCounts(1, 0, 1); assertDatabaseCounts(1); } @@ -160,7 +153,7 @@ public void testDeduplicatesAccessionsFromInput() { List svesToWrite = Arrays.asList( createSve(5000000003L, 1100, "C", "T", LocalDateTime.now(), "GCA_000000001.1"), createSve(5000000003L, 1100, "A", "T", LocalDateTime.now(), "GCA_000000001.1")); - writer.write(svesToWrite); + writer.write(new Chunk<>(svesToWrite)); assertRemappingIngestCounts(1, 0, 1); assertDatabaseCounts(1); } @@ -172,7 +165,7 @@ public void testDiscardsDuplicateHashes() { List svesToWrite = Collections.singletonList( createSve(5000000005L, 1100, "C", "T", LocalDateTime.now(), "GCA_000000001.1")); - writer.write(svesToWrite); + writer.write(new Chunk<>(svesToWrite)); assertRemappingIngestCounts(0, 0, 1); assertDatabaseCounts(1); } @@ -187,7 +180,7 @@ public void testDeduplicatesHashesFromInput() { createSve(5000000003L, 1100, "C", "T", LocalDateTime.now(), "GCA_000000001.1"), // Duplicate hash but different accession => discard operation createSve(5000000004L, 1100, "C", "T", LocalDateTime.now(), "GCA_000000001.1")); - writer.write(svesToWrite); + writer.write(new Chunk<>(svesToWrite)); assertRemappingIngestCounts(0, 1, 1); assertDatabaseCounts(1); } @@ -201,7 +194,7 @@ public void testDuplicateHashAndAccessionButDifferentSVEs() { SubmittedVariantEntity duplicateSve = createSve(5000000003L, 1100, "C", "T", LocalDateTime.now(), "GCA_000000001.1"); duplicateSve.setBackPropagatedVariantAccession(12345L); List svesToWrite = Collections.singletonList(duplicateSve); - writer.write(svesToWrite); + writer.write(new Chunk<>(svesToWrite)); assertRemappingIngestCounts(0, 0, 1); assertDatabaseCounts(1); } @@ -212,7 +205,7 @@ public void testIdempotentWrites() { List svesToWrite = Collections.singletonList( createSve(5000000002L, 1100, "C", "T", LocalDateTime.now(), null)); - writer.write(svesToWrite); + writer.write(new Chunk<>(svesToWrite)); assertRemappingIngestCounts(1, 0, 1); Set sveAfterFirstWrite = new HashSet<>( @@ -221,7 +214,7 @@ public void testIdempotentWrites() { mongoTemplate.findAll(SubmittedVariantOperationEntity.class)); remappingIngestCounts.resetCounts(); - writer.write(svesToWrite); + writer.write(new Chunk<>(svesToWrite)); assertRemappingIngestCounts(0, 1, 0); Set sveAfterSecondWrite = new HashSet<>( @@ -249,7 +242,7 @@ public void testDuplicateAccessionInDb_discardFromDb() { List firstBatch = Collections.singletonList( createSve(5000000003L, 1400, "C", "T", now, "GCA_000000001.1") ); - writer.write(firstBatch); + writer.write(new Chunk<>(firstBatch)); assertRemappingIngestCounts(0, 0, 2); assertDatabaseCounts(1); @@ -257,7 +250,7 @@ public void testDuplicateAccessionInDb_discardFromDb() { createSve(5000000003L, 1400, "C", "T", now, "GCA_000000001.1"), createSve(5000000004L, 1400, "C", "T", now, "GCA_000000001.1") ); - writer.write(secondBatch); + writer.write(new Chunk<>(secondBatch)); assertRemappingIngestCounts(0, 1, 3); assertDatabaseCounts(1); } @@ -281,7 +274,7 @@ public void testDuplicateAccessionInDb_skipFromInput() { createSve(5000000003L, 1400, "C", "T", now, "GCA_000000001.1"), createSve(5000000004L, 1400, "C", "T", now, "GCA_000000001.1") ); - writer.write(svesToWrite); + writer.write(new Chunk<>(svesToWrite)); assertRemappingIngestCounts(1, 2, 0); assertDatabaseCounts(3); } diff --git a/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/jobs/IngestRemappedVariantsFromVcfJobConfigurationTest.java b/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/jobs/IngestRemappedVariantsFromVcfJobConfigurationTest.java index c585da7bf..232ec7218 100644 --- a/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/jobs/IngestRemappedVariantsFromVcfJobConfigurationTest.java +++ b/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/jobs/IngestRemappedVariantsFromVcfJobConfigurationTest.java @@ -15,65 +15,66 @@ */ package uk.ac.ebi.eva.remapping.ingest.configuration.batch.jobs; -import com.lordofthejars.nosqlunit.annotation.UsingDataSet; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; -import org.junit.After; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.batch.core.BatchStatus; import org.springframework.batch.core.JobExecution; import org.springframework.batch.core.StepExecution; import org.springframework.batch.test.JobLauncherTestUtils; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.ApplicationContext; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.core.io.ResourceLoader; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.data.mongodb.core.query.Criteria; import org.springframework.data.mongodb.core.query.Query; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; - +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; +import uk.ac.ebi.eva.accession.core.utils.MongoTestDataLoader; import uk.ac.ebi.eva.remapping.ingest.batch.tasklets.RemappingMetadata; +import uk.ac.ebi.eva.remapping.ingest.test.configuration.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.remapping.ingest.test.configuration.BatchTestConfiguration; -import uk.ac.ebi.eva.remapping.ingest.test.rule.FixSpringMongoDbRule; +import uk.ac.ebi.eva.remapping.ingest.test.configuration.MongoTestConfiguration; import java.util.Arrays; import java.util.Collection; import java.util.List; import java.util.stream.Collectors; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static uk.ac.ebi.eva.remapping.ingest.configuration.BeanNames.INGEST_REMAPPED_VARIANTS_FROM_VCF_STEP; import static uk.ac.ebi.eva.remapping.ingest.configuration.BeanNames.STORE_REMAPPING_METADATA_STEP; +import static uk.ac.ebi.eva.remapping.ingest.test.configuration.BatchTestConfiguration.JOB_INGEST_REMAPPED_VARIANTS_FROM_VCF_JOB; -@RunWith(SpringRunner.class) -@ContextConfiguration(classes = {BatchTestConfiguration.class}) +@ExtendWith(SpringExtension.class) +@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class, + BatchJobRepositoryTestConfiguration.class}) @TestPropertySource("classpath:ingest-remapped-variants.properties") -@UsingDataSet(locations = {"/test-data/submittedVariantEntity.json"}) -public class IngestRemappedVariantsFromVcfJobConfigurationTest { - - private static final String TEST_DB = "test-ingest-remapping"; +public class IngestRemappedVariantsFromVcfJobConfigurationTest extends MongoTestContainerHelper { @Autowired + @Qualifier(JOB_INGEST_REMAPPED_VARIANTS_FROM_VCF_JOB) private JobLauncherTestUtils jobLauncherTestUtils; @Autowired private MongoTemplate mongoTemplate; - //Required by nosql-unit @Autowired - private ApplicationContext applicationContext; + private ResourceLoader resourceLoader; - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName(TEST_DB).build()); + @BeforeEach + public void setUp() { + mongoTemplate.getDb().drop(); + new MongoTestDataLoader(mongoTemplate, resourceLoader).load("/test-data/submittedVariantEntity.json"); + } - @After + @AfterEach public void tearDown() { mongoTemplate.getDb().drop(); } @@ -83,7 +84,7 @@ public void tearDown() { public void jobFromVcf() throws Exception { JobExecution jobExecution = jobLauncherTestUtils.launchJob(); List expectedSteps = Arrays.asList(STORE_REMAPPING_METADATA_STEP, - INGEST_REMAPPED_VARIANTS_FROM_VCF_STEP); + INGEST_REMAPPED_VARIANTS_FROM_VCF_STEP); assertStepsExecuted(expectedSteps, jobExecution); assertEquals(BatchStatus.COMPLETED, jobExecution.getStatus()); assertMetadataAssociatedToSubmittedVariants(); @@ -104,9 +105,9 @@ private void assertMetadataAssociatedToSubmittedVariants() { List variantsRemapped = mongoTemplate.find(remappedQuery, SubmittedVariantEntity.class); long variantsWithMetatada = variantsRemapped.stream() - .filter(x -> x.getRemappingId() != null && - x.getRemappingId().equals(remappingId)) - .count(); + .filter(x -> x.getRemappingId() != null && + x.getRemappingId().equals(remappingId)) + .count(); assertEquals(4, variantsWithMetatada); } } \ No newline at end of file diff --git a/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/steps/IngestRemappedFromVcfStepConfigurationTest.java b/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/steps/IngestRemappedFromVcfStepConfigurationTest.java index f74407a71..931ba1175 100644 --- a/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/steps/IngestRemappedFromVcfStepConfigurationTest.java +++ b/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/steps/IngestRemappedFromVcfStepConfigurationTest.java @@ -16,75 +16,60 @@ */ package uk.ac.ebi.eva.remapping.ingest.configuration.batch.steps; -import com.lordofthejars.nosqlunit.annotation.UsingDataSet; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.batch.core.BatchStatus; import org.springframework.batch.core.JobExecution; import org.springframework.batch.test.JobLauncherTestUtils; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.ApplicationContext; +import org.springframework.core.io.ResourceLoader; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.data.mongodb.core.query.Criteria; import org.springframework.data.mongodb.core.query.Query; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; -import uk.ac.ebi.ampt2d.commons.accession.hashing.SHA1HashingFunction; - -import uk.ac.ebi.eva.accession.core.model.ISubmittedVariant; +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; -import uk.ac.ebi.eva.accession.core.summary.SubmittedVariantSummaryFunction; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; +import uk.ac.ebi.eva.accession.core.utils.MongoTestDataLoader; import uk.ac.ebi.eva.remapping.ingest.configuration.BeanNames; +import uk.ac.ebi.eva.remapping.ingest.test.configuration.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.remapping.ingest.test.configuration.BatchTestConfiguration; -import uk.ac.ebi.eva.remapping.ingest.test.rule.FixSpringMongoDbRule; +import uk.ac.ebi.eva.remapping.ingest.test.configuration.MongoTestConfiguration; import java.util.List; -import java.util.function.Function; -import static junit.framework.TestCase.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; -@RunWith(SpringRunner.class) -@ContextConfiguration(classes = {BatchTestConfiguration.class}) +@ExtendWith(SpringExtension.class) +@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class, + BatchJobRepositoryTestConfiguration.class}) @TestPropertySource("classpath:ingest-remapped-variants.properties") -@UsingDataSet(locations = {"/test-data/submittedVariantEntity.json"}) -public class IngestRemappedFromVcfStepConfigurationTest { - - private static final String TEST_DB = "test-ingest-remapping"; +public class IngestRemappedFromVcfStepConfigurationTest extends MongoTestContainerHelper { private static final String SUBMITTED_VARIANT_COLLECTION = "submittedVariantEntity"; public static final String REMAPPED_FROM = "GCA_000000001.1"; - private Function hashingFunction; - @Autowired private JobLauncherTestUtils jobLauncherTestUtils; @Autowired private MongoTemplate mongoTemplate; - //Required by nosql-unit @Autowired - private ApplicationContext applicationContext; + private ResourceLoader resourceLoader; - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName(TEST_DB).build()); - - @Before - public void setUp() throws Exception { - hashingFunction = new SubmittedVariantSummaryFunction().andThen(new SHA1HashingFunction()); + @BeforeEach + public void setUp() { + mongoTemplate.getDb().drop(); + new MongoTestDataLoader(mongoTemplate, resourceLoader).load("/test-data/submittedVariantEntity.json"); } - @After + @AfterEach public void tearDown() { mongoTemplate.getDb().drop(); } @@ -96,14 +81,14 @@ public void runStep() { assertEquals(7, mongoTemplate.getCollection(SUBMITTED_VARIANT_COLLECTION).countDocuments()); JobExecution jobExecution = jobLauncherTestUtils.launchStep(BeanNames.INGEST_REMAPPED_VARIANTS_FROM_VCF_STEP); - Assert.assertEquals(BatchStatus.COMPLETED, jobExecution.getStatus()); + assertEquals(BatchStatus.COMPLETED, jobExecution.getStatus()); //Documents in the database after the ingestion assertEquals(11, mongoTemplate.getCollection(SUBMITTED_VARIANT_COLLECTION).countDocuments()); Query remappedVariantsQuery = new Query(Criteria.where("remappedFrom").is(REMAPPED_FROM)); List remappedVariants = mongoTemplate.find(remappedVariantsQuery, - SubmittedVariantEntity.class); + SubmittedVariantEntity.class); assertEquals(6, remappedVariants.size()); diff --git a/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/steps/IngestRemappedFromVcfStepDbsnpConfigurationTest.java b/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/steps/IngestRemappedFromVcfStepDbsnpConfigurationTest.java index d53ce1a8f..377c3ee6d 100644 --- a/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/steps/IngestRemappedFromVcfStepDbsnpConfigurationTest.java +++ b/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/steps/IngestRemappedFromVcfStepDbsnpConfigurationTest.java @@ -16,87 +16,79 @@ */ package uk.ac.ebi.eva.remapping.ingest.configuration.batch.steps; -import com.lordofthejars.nosqlunit.annotation.UsingDataSet; -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; -import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.batch.core.BatchStatus; import org.springframework.batch.core.JobExecution; import org.springframework.batch.test.JobLauncherTestUtils; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.ApplicationContext; +import org.springframework.core.io.ResourceLoader; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.data.mongodb.core.query.Criteria; import org.springframework.data.mongodb.core.query.Query; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; -import uk.ac.ebi.ampt2d.commons.accession.hashing.SHA1HashingFunction; - -import uk.ac.ebi.eva.accession.core.model.ISubmittedVariant; +import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpSubmittedVariantEntity; -import uk.ac.ebi.eva.accession.core.summary.SubmittedVariantSummaryFunction; +import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; +import uk.ac.ebi.eva.accession.core.utils.MongoTestDataLoader; import uk.ac.ebi.eva.remapping.ingest.configuration.BeanNames; +import uk.ac.ebi.eva.remapping.ingest.test.configuration.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.remapping.ingest.test.configuration.BatchTestConfiguration; -import uk.ac.ebi.eva.remapping.ingest.test.rule.FixSpringMongoDbRule; +import uk.ac.ebi.eva.remapping.ingest.test.configuration.MongoTestConfiguration; import java.util.List; -import java.util.function.Function; -import static junit.framework.TestCase.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; -@RunWith(SpringRunner.class) -@ContextConfiguration(classes = {BatchTestConfiguration.class}) +@ExtendWith(SpringExtension.class) +@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class, + BatchJobRepositoryTestConfiguration.class}) @TestPropertySource("classpath:ingest-remapped-variants-dbsnp.properties") -@UsingDataSet(locations = {"/test-data/dbsnpSubmittedVariantEntity.json"}) -public class IngestRemappedFromVcfStepDbsnpConfigurationTest { - - private static final String TEST_DB = "test-ingest-remapping"; +public class IngestRemappedFromVcfStepDbsnpConfigurationTest extends MongoTestContainerHelper { private static final String DBSNP_SUBMITTED_VARIANT_COLLECTION = "dbsnpSubmittedVariantEntity"; public static final String REMAPPED_FROM = "GCA_000000001.1"; - private Function hashingFunction; - @Autowired private JobLauncherTestUtils jobLauncherTestUtils; @Autowired private MongoTemplate mongoTemplate; - //Required by nosql-unit @Autowired - private ApplicationContext applicationContext; + private ResourceLoader resourceLoader; - @Rule - public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( - MongoDbConfigurationBuilder.mongoDb().databaseName(TEST_DB).build()); + @BeforeEach + public void setUp() { + mongoTemplate.getDb().drop(); + new MongoTestDataLoader(mongoTemplate, resourceLoader).load("/test-data/dbsnpSubmittedVariantEntity.json"); + } - @Before - public void setUp() throws Exception { - hashingFunction = new SubmittedVariantSummaryFunction().andThen(new SHA1HashingFunction()); + @AfterEach + public void tearDown() { + mongoTemplate.getDb().drop(); } + @Test @DirtiesContext public void runStepDbsnp() { assertEquals(1, mongoTemplate.getCollection(DBSNP_SUBMITTED_VARIANT_COLLECTION).countDocuments()); JobExecution jobExecution = jobLauncherTestUtils.launchStep(BeanNames.INGEST_REMAPPED_VARIANTS_FROM_VCF_STEP); - Assert.assertEquals(BatchStatus.COMPLETED, jobExecution.getStatus()); + assertEquals(BatchStatus.COMPLETED, jobExecution.getStatus()); //Documents in the database after the ingestion assertEquals(2, mongoTemplate.getCollection(DBSNP_SUBMITTED_VARIANT_COLLECTION).countDocuments()); Query remappedVariantsQuery = new Query(Criteria.where("remappedFrom").is(REMAPPED_FROM)); List remappedVariants = mongoTemplate.find(remappedVariantsQuery, - DbsnpSubmittedVariantEntity.class); + DbsnpSubmittedVariantEntity.class); assertEquals(1, remappedVariants.size()); diff --git a/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/test/configuration/BatchJobRepositoryTestConfiguration.java b/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/test/configuration/BatchJobRepositoryTestConfiguration.java new file mode 100644 index 000000000..cfa3352cb --- /dev/null +++ b/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/test/configuration/BatchJobRepositoryTestConfiguration.java @@ -0,0 +1,22 @@ +package uk.ac.ebi.eva.remapping.ingest.test.configuration; + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Primary; +import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseBuilder; +import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseType; + +import javax.sql.DataSource; + +@Configuration +public class BatchJobRepositoryTestConfiguration { + + @Bean + @Primary + public DataSource dataSource() { + return new EmbeddedDatabaseBuilder() + .setType(EmbeddedDatabaseType.HSQL) + .addScript("org/springframework/batch/core/schema-hsqldb.sql") + .build(); + } +} \ No newline at end of file diff --git a/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/test/configuration/BatchTestConfiguration.java b/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/test/configuration/BatchTestConfiguration.java index 9471b8897..a7065192a 100644 --- a/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/test/configuration/BatchTestConfiguration.java +++ b/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/test/configuration/BatchTestConfiguration.java @@ -16,15 +16,16 @@ */ package uk.ac.ebi.eva.remapping.ingest.test.configuration; +import org.springframework.batch.core.Job; +import org.springframework.batch.core.launch.JobLauncher; +import org.springframework.batch.core.repository.JobRepository; import org.springframework.batch.test.JobLauncherTestUtils; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; -import org.springframework.boot.autoconfigure.batch.BatchProperties; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Import; import org.springframework.core.io.ResourceLoader; -import org.springframework.transaction.PlatformTransactionManager; - import uk.ac.ebi.eva.commons.batch.job.JobExecutionApplicationListener; import uk.ac.ebi.eva.remapping.ingest.configuration.RemappingMetadataConfiguration; import uk.ac.ebi.eva.remapping.ingest.configuration.batch.io.IngestRemappedSubmittedVariantsWriterConfiguration; @@ -36,7 +37,7 @@ import uk.ac.ebi.eva.remapping.ingest.configuration.batch.steps.StoreRemappingMetadataStepConfiguration; import uk.ac.ebi.eva.remapping.ingest.configuration.policies.ChunkSizeCompletionPolicyConfiguration; -import javax.sql.DataSource; +import static uk.ac.ebi.eva.remapping.ingest.configuration.BeanNames.INGEST_REMAPPED_VARIANTS_FROM_VCF_JOB; @EnableAutoConfiguration @Import({IngestRemappedFromVcfStepConfiguration.class, @@ -49,22 +50,19 @@ ChunkSizeCompletionPolicyConfiguration.class, RemappingMetadataConfiguration.class}) public class BatchTestConfiguration { - - @Autowired - private BatchProperties properties; + public static final String JOB_INGEST_REMAPPED_VARIANTS_FROM_VCF_JOB = "JOB_INGEST_REMAPPED_VARIANTS_FROM_VCF_JOB"; @Autowired private ResourceLoader resourceLoader; - @Autowired - private DataSource dataSource; - - @Autowired - private PlatformTransactionManager platformTransactionManager; - - @Bean - public JobLauncherTestUtils jobLauncherTestUtils() { - return new JobLauncherTestUtils(); + @Bean(JOB_INGEST_REMAPPED_VARIANTS_FROM_VCF_JOB) + public JobLauncherTestUtils jobLauncherTestUtilsFromMongo(JobLauncher jobLauncher, JobRepository jobRepository, + @Qualifier(INGEST_REMAPPED_VARIANTS_FROM_VCF_JOB) Job job) { + JobLauncherTestUtils utils = new JobLauncherTestUtils(); + utils.setJobLauncher(jobLauncher); + utils.setJobRepository(jobRepository); + utils.setJob(job); + return utils; } @Bean diff --git a/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/test/configuration/MongoTestConfiguration.java b/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/test/configuration/MongoTestConfiguration.java index 95135e52c..2cab74199 100644 --- a/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/test/configuration/MongoTestConfiguration.java +++ b/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/test/configuration/MongoTestConfiguration.java @@ -15,12 +15,30 @@ */ package uk.ac.ebi.eva.remapping.ingest.test.configuration; +import com.mongodb.ReadPreference; +import com.mongodb.WriteConcern; +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoClients; +import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.domain.EntityScan; import org.springframework.boot.test.autoconfigure.data.mongo.AutoConfigureDataMongo; +import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Primary; +import org.springframework.data.mongodb.MongoDatabaseFactory; import org.springframework.data.mongodb.config.EnableMongoAuditing; +import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.data.mongodb.core.SimpleMongoClientDatabaseFactory; +import org.springframework.data.mongodb.core.WriteResultChecking; +import org.springframework.data.mongodb.core.convert.DefaultDbRefResolver; +import org.springframework.data.mongodb.core.convert.DefaultMongoTypeMapper; +import org.springframework.data.mongodb.core.convert.MappingMongoConverter; +import org.springframework.data.mongodb.core.convert.MongoCustomConversions; +import org.springframework.data.mongodb.core.mapping.MongoMappingContext; import org.springframework.data.mongodb.repository.config.EnableMongoRepositories; +import java.util.Collections; + @Configuration @EntityScan(basePackages = {"uk.ac.ebi.eva.accession.core.persistence"}) @EnableMongoRepositories(basePackages = { @@ -29,4 +47,56 @@ @EnableMongoAuditing @AutoConfigureDataMongo public class MongoTestConfiguration { + + @Value("${spring.data.mongodb.uri}") + private String mongoUri; + + @Value("${spring.data.mongodb.database}") + private String database; + + @Value("${mongodb.read-preference:primary}") + private String readPreference; + + @Bean + @Primary + public MongoClient mongoClient() { + return MongoClients.create(mongoUri); + } + + @Bean("primaryFactory") + @Primary + public MongoDatabaseFactory mongoDbFactory() { + return new SimpleMongoClientDatabaseFactory(mongoClient(), database); + } + + @Bean + @Primary + public MongoCustomConversions mongoCustomConversions() { + return new MongoCustomConversions(Collections.emptyList()); + } + + @Bean + @Primary + public MongoMappingContext mongoMappingContext(MongoCustomConversions conversions) { + MongoMappingContext context = new MongoMappingContext(); + context.setSimpleTypeHolder(conversions.getSimpleTypeHolder()); + context.setAutoIndexCreation(false); + return context; + } + + @Bean + @Primary + public MongoTemplate mongoTemplate(MongoMappingContext mongoMappingContext, + MongoCustomConversions conversions) { + MappingMongoConverter converter = new MappingMongoConverter( + new DefaultDbRefResolver(mongoDbFactory()), mongoMappingContext); + converter.setCustomConversions(conversions); + converter.setTypeMapper(new DefaultMongoTypeMapper(null)); + converter.afterPropertiesSet(); + MongoTemplate mongoTemplate = new MongoTemplate(mongoDbFactory(), converter); + mongoTemplate.setReadPreference(ReadPreference.valueOf(readPreference)); + mongoTemplate.setWriteConcern(WriteConcern.MAJORITY); + mongoTemplate.setWriteResultChecking(WriteResultChecking.EXCEPTION); + return mongoTemplate; + } } diff --git a/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/test/rule/FixSpringMongoDbRule.java b/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/test/rule/FixSpringMongoDbRule.java deleted file mode 100644 index 9386f9b1d..000000000 --- a/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/test/rule/FixSpringMongoDbRule.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright 2021 EMBL - European Bioinformatics Institute - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package uk.ac.ebi.eva.remapping.ingest.test.rule; - -import com.lordofthejars.nosqlunit.mongodb.MongoDbConfiguration; -import com.lordofthejars.nosqlunit.mongodb.SpringMongoDbRule; - -/** - * Temporary fix until nosql unit rc-6 or final is released - */ -public class FixSpringMongoDbRule extends SpringMongoDbRule { - - public FixSpringMongoDbRule(MongoDbConfiguration mongoDbConfiguration) { - super(mongoDbConfiguration); - } - - public FixSpringMongoDbRule(MongoDbConfiguration mongoDbConfiguration, Object object) { - super(mongoDbConfiguration, object); - } - - @Override - public void close() { - // DO NOT CLOSE the connection (Spring will do it when destroying the context) - } -} diff --git a/eva-remapping-ingest/src/test/resources/ingest-remapped-variants-dbsnp.properties b/eva-remapping-ingest/src/test/resources/ingest-remapped-variants-dbsnp.properties index a4baca6b4..07145cff2 100644 --- a/eva-remapping-ingest/src/test/resources/ingest-remapped-variants-dbsnp.properties +++ b/eva-remapping-ingest/src/test/resources/ingest-remapped-variants-dbsnp.properties @@ -10,9 +10,6 @@ parameters.chunkSize=2 build.version=|project.version| spring.data.mongodb.database=test-ingest-remapping -spring.data.mongodb.host=|eva.mongo.host.test| -spring.data.mongodb.password= -spring.data.mongodb.port=27017 mongodb.read-preference=primary # See https://github.com/spring-projects/spring-boot/wiki/Spring-Boot-2.1-Release-Notes#bean-overriding diff --git a/eva-remapping-ingest/src/test/resources/ingest-remapped-variants.properties b/eva-remapping-ingest/src/test/resources/ingest-remapped-variants.properties index c2ee2a787..864bdf764 100644 --- a/eva-remapping-ingest/src/test/resources/ingest-remapped-variants.properties +++ b/eva-remapping-ingest/src/test/resources/ingest-remapped-variants.properties @@ -10,9 +10,6 @@ parameters.chunkSize=2 build.version=|project.version| spring.data.mongodb.database=test-ingest-remapping -spring.data.mongodb.host=|eva.mongo.host.test| -spring.data.mongodb.password= -spring.data.mongodb.port=27017 mongodb.read-preference=primary # See https://github.com/spring-projects/spring-boot/wiki/Spring-Boot-2.1-Release-Notes#bean-overriding diff --git a/pom.xml b/pom.xml index c8015f570..9843ab4d1 100644 --- a/pom.xml +++ b/pom.xml @@ -1,10 +1,9 @@ 4.0.0 - uk.ac.ebi.eva eva-accession - 0.6.58-SNAPSHOT + ${revision} pom eva-accession @@ -23,16 +22,20 @@ + 0.6.58-SNAPSHOT + 21 UTF-8 - 4.13 - 0.8.10 - 0.7.21 + 1.0.0-SNAPSHOT + 1.0.0-SNAPSHOT + 4.0.0 + 1.0.0-SNAPSHOT + 2.8.17 org.springframework.boot spring-boot-starter-parent - 2.1.0.RELEASE + 3.4.13 @@ -40,18 +43,49 @@ uk.ac.ebi.eva eva-accession-core - 0.6.58-SNAPSHOT + ${revision} uk.ac.ebi.eva - eva-accession-clustering - 0.6.58-SNAPSHOT + eva-accession-ws + ${revision} + + + uk.ac.ebi.eva + eva-accession-pipeline + ${revision} + + + uk.ac.ebi.eva + eva-accession-release + ${revision} uk.ac.ebi.eva eva-remapping-get-source - 0.6.58-SNAPSHOT + ${revision} + + + uk.ac.ebi.eva + eva-accession-deprecate + ${revision} + + + uk.ac.ebi.eva + eva-accession-import-dbsnp2 + ${revision} + + uk.ac.ebi.eva + eva-accession-clustering + ${revision} + + + uk.ac.ebi.eva + eva-remapping-ingest + ${revision} + + uk.ac.ebi.ampt2d accession-commons-mongodb @@ -85,38 +119,22 @@ uk.ac.ebi.eva eva-metrics - 0.0.4 - - - org.postgresql - postgresql - 42.7.5 - - - com.lordofthejars - nosqlunit-mongodb - 1.0.0-rc.5 - test + ${eva-metrics-version} com.github.samtools htsjdk - 2.14.3 + ${samtools-htsjdk-version} com.thoughtworks.xstream xstream - 1.4.9 + 1.4.21 - io.springfox - springfox-swagger2 - 2.8.0 - - - io.springfox - springfox-swagger-ui - 2.8.0 + org.springdoc + springdoc-openapi-starter-webmvc-ui + ${springdoc.version} @@ -135,10 +153,30 @@ test + + org.postgresql + postgresql + + org.hsqldb hsqldb - 2.3.4 + test + + + + org.springframework.boot + spring-boot-testcontainers + test + + + org.testcontainers + mongodb + test + + + org.testcontainers + junit-jupiter test @@ -170,11 +208,27 @@ + + + + org.apache.maven.plugins + maven-surefire-plugin + + false + + --add-opens java.base/java.time=ALL-UNNAMED + --add-opens java.base/java.lang=ALL-UNNAMED + --add-opens java.base/java.util=ALL-UNNAMED + + + + + org.apache.maven.plugins maven-resources-plugin - 2.6 + 3.3.0 @@ -186,21 +240,16 @@ org.apache.maven.plugins maven-compiler-plugin - 1.8 - 1.8 - - - - org.apache.maven.plugins - maven-surefire-plugin - - false + ${java.version} + true + true + UTF-8 - pl.project13.maven - git-commit-id-plugin - 2.2.5 + io.github.git-commit-id + git-commit-id-maven-plugin + 9.0.1 ^git.tags$ @@ -212,7 +261,32 @@ + + org.codehaus.mojo + flatten-maven-plugin + 1.6.0 + + true + resolveCiFriendliesOnly + + + + flatten + process-resources + + flatten + + + + flatten.clean + clean + + clean + + + + - + \ No newline at end of file From 61d650048a3a9bb801208f4e73693f170db98152 Mon Sep 17 00:00:00 2001 From: nitin-ebi <79518737+nitin-ebi@users.noreply.github.com> Date: Thu, 4 Jun 2026 15:08:34 +0100 Subject: [PATCH 08/10] EVA-4177 Remove external job tracker - use in memory one (#502) --- eva-accession-clustering/pom.xml | 11 + .../eva/accession/clustering/Application.java | 19 +- .../jobs/BackPropagateRSJobConfiguration.java | 5 - ...erUnclusteredVariantsJobConfiguration.java | 4 - .../ClusteringFromMongoJobConfiguration.java | 11 +- ...emappedVariantsWithRSJobConfiguration.java | 4 - .../RSAccessionRecoveryJobConfiguration.java | 4 - .../jobs/StudyClusteringJobConfiguration.java | 4 - .../jobs/qc/ClusteringQCJobConfiguration.java | 11 +- .../qc/DuplicateRSAccQCJobConfiguration.java | 4 - ...ewClusteredVariantsQCJobConfiguration.java | 9 +- ...AccessioningShutdownStepConfiguration.java | 7 +- .../ClusteringFromMongoStepConfiguration.java | 27 ++- .../RSAccessionRecoveryStepConfiguration.java | 4 +- .../qc/DuplicateRSAccQCStepConfiguration.java | 6 +- .../parameters/InputParameters.java | 10 - .../runner/ClusteringCommandLineRunner.java | 15 +- .../src/main/resources/application.properties | 6 - .../batch/io/BackPropagatedRSWriterTest.java | 6 +- .../batch/io/ClusteringMongoReaderTest.java | 2 +- .../RSMergeAndSplitCandidatesReaderTest.java | 7 +- .../batch/io/RSMergeWriterTest.java | 5 +- .../batch/io/RSSplitWriterTest.java | 26 ++- .../batch/io/SSSplitWriterTest.java | 24 +- .../io/StudyClusteringMongoReaderTest.java | 2 +- .../IssueAccessionClusteringWriterTest.java | 27 ++- .../MergeAccessionClusteringWriterTest.java | 8 +- .../RemappedVariantsClusteringWriterTest.java | 27 ++- .../ReuseAccessionClusteringWriterTest.java | 27 ++- ...ClusteringVariantJobConfigurationTest.java | 22 +- .../batch/jobs/RSAccessionRecoveryTest.java | 35 ++- ...DuplicateRSAccQCStepConfigurationTest.java | 26 ++- ...usteredVariantsQCJobConfigurationTest.java | 7 +- ...lusteringVariantStepConfigurationTest.java | 22 +- .../ClusteringCommandLineRunnerTest.java | 94 ++------ .../BatchJobRepositoryTestConfiguration.java | 22 -- .../configuration/BatchTestConfiguration.java | 13 +- .../configuration/MongoTestConfiguration.java | 102 --------- .../RSAccessionRecoveryTestConfiguration.java | 2 - .../resources/backpropagation-test.properties | 4 + .../clustering-issuance-test.properties | 5 + .../clustering-pipeline-test.properties | 5 + .../resources/clustering-qc-test.properties | 5 + .../clustering-writer-test.properties | 5 + .../duplicate-rs-acc-qc-test.properties | 5 + .../resources/merge-split-test.properties | 5 + .../rs-accession-recovery.properties | 12 +- .../test-data/contiguous_id_blocks_schema.sql | 4 + eva-accession-core/pom.xml | 9 +- .../accession/core/GenericApplication.java | 15 +- ...iguousIdBlocksDataSourceConfiguration.java | 50 ++++ .../InMemoryBatchConfiguration.java | 50 ++++ .../core/configuration/NoOpJobExplorer.java | 65 ++++++ .../core/runner/CommandLineRunnerUtils.java | 96 -------- ...ClusteredVariantDeprecationWriterTest.java | 6 +- ...SubmittedVariantDeprecationWriterTest.java | 8 +- ...ClusteredVariantEntityIntegrationTest.java | 2 - ...SubmittedVariantEntityIntegrationTest.java | 2 - ...ttedVariantAccessioningRepositoryTest.java | 4 +- ...usteredVariantAccessioningServiceTest.java | 4 +- ...bmittedVariantAccessioningServiceTest.java | 26 ++- .../configuration/JPATestConfiguration.java | 41 ---- .../properties/ss-accession-test.properties | 5 + .../properties/ss-deprecation-test.properties | 5 + .../test-data/contiguous_id_blocks_schema.sql | 14 ++ eva-accession-deprecate/pom.xml | 11 + .../eva/accession/deprecate/Application.java | 19 +- ...tudySubmittedVariantsJobConfiguration.java | 4 - ...ittedVariantsFromFileJobConfiguration.java | 4 - ...udySubmittedVariantsStepConfiguration.java | 5 +- ...ttedVariantsFromFileStepConfiguration.java | 5 +- .../src/main/resources/application.properties | 6 - .../io/StudySubmittedVariantsReaderTest.java | 2 +- ...SubmittedVariantsJobConfigurationTest.java | 37 +-- ...dVariantsFromFileJobConfigurationTest.java | 37 +-- ...ubmittedVariantsStepConfigurationTest.java | 6 +- ...VariantsFromFileStepConfigurationTest.java | 6 +- .../BatchJobRepositoryTestConfiguration.java | 22 -- .../configuration/BatchTestConfiguration.java | 14 +- .../configuration/MongoTestConfiguration.java | 105 --------- ...bmitted-variants-from-file-test.properties | 5 + .../study-submitted-variants-test.properties | 5 + eva-accession-import-dbsnp2/pom.xml | 10 +- .../ebi/eva/accession/dbsnp2/Application.java | 16 +- ...portDbsnpJsonVariantsJobConfiguration.java | 4 - ...ortDbsnpJsonVariantsStepConfiguration.java | 6 +- .../dbsnp2/parameters/InputParameters.java | 19 +- ...tVariantsJobLauncherCommandLineRunner.java | 16 +- .../src/main/resources/application.properties | 1 - .../DbsnpJsonClusteredVariantsWriterTest.java | 2 +- ...DbsnpJsonVariantsJobConfigurationTest.java | 6 +- ...bsnpJsonVariantsStepConfigurationTest.java | 6 +- ...iantsJobLauncherCommandLineRunnerTest.java | 139 +----------- .../BatchJobRepositoryTestConfiguration.java | 22 -- .../dbsnp2/test/BatchTestConfiguration.java | 8 +- .../dbsnp2/test/MongoTestConfiguration.java | 102 --------- .../src/test/resources/application.properties | 1 - eva-accession-pipeline/pom.xml | 5 + .../eva/accession/pipeline/Application.java | 20 +- .../DuplicateSSAccQCJobConfiguration.java | 4 - .../QCSubsnpAccessionsJobConfiguration.java | 4 - .../SSAccessionRecoveryJobConfiguration.java | 4 - .../SubsnpAccessionsJobConfiguration.java | 4 - ...AccessioningShutdownStepConfiguration.java | 7 +- .../steps/BuildReportStepConfiguration.java | 7 +- .../DuplicateSSAccQCStepConfiguration.java | 6 +- .../QCSubsnpAccessionsStepConfiguration.java | 6 +- .../SSAccessionRecoveryStepConfiguration.java | 2 - .../SubsnpAccessionsStepConfiguration.java | 6 +- .../pipeline/parameters/InputParameters.java | 10 - ...AccessionJobLauncherCommandLineRunner.java | 16 +- .../src/main/resources/application.properties | 6 - .../batch/io/AccessionWriterTest.java | 25 +- ...fSubsnpAccessionsJobConfigurationTest.java | 22 +- ...eSubsnpAccessionsJobConfigurationTest.java | 22 +- ...reateSubsnpAccessionsRecoverStateTest.java | 31 ++- ...nsRecoveringStateJobConfigurationTest.java | 24 +- .../DuplicateSSAccQCJobConfigurationTest.java | 6 +- ...CSubsnpAccessionsJobConfigurationTest.java | 6 +- ...SubsnpAccessionsStepConfigurationTest.java | 22 +- ...SubsnpAccessionsStepConfigurationTest.java | 6 +- ...ssionJobLauncherCommandLineRunnerTest.java | 165 ++------------ .../runner/JobFailureBlocksReleasedTest.java | 23 +- .../pipeline/runner/RestartFailedJobTest.java | 214 ------------------ .../runner/SSAccessionRecoveryTest.java | 38 +++- .../BatchJobRepositoryTestConfiguration.java | 22 -- .../pipeline/test/BatchTestConfiguration.java | 11 +- .../pipeline/test/MongoTestConfiguration.java | 102 --------- .../SSAccessionRecoveryTestConfiguration.java | 2 - ...ccession-pipeline-interval-test.properties | 5 + ...ion-pipeline-recover-state-test.properties | 12 +- ...accession-pipeline-recover-test.properties | 12 +- .../accession-pipeline-test.properties | 5 + .../duplicate-ss-acc-qc-test.properties | 5 + .../qc-accession-pipeline-test.properties | 5 + .../ss-accession-recovery.properties | 12 +- .../test-data/contiguous_id_blocks_schema.sql | 6 +- .../eva/accession/release/Application.java | 11 +- .../configuration/RunnerConfiguration.java | 25 ++ ...ccessionReleaseFromDBJobConfiguration.java | 4 - .../DumpRSAccessionsJobConfiguration.java | 5 - ...ccessionReleaseFromDBJobConfiguration.java | 4 - ...cessionReleaseFromDBStepConfiguration.java | 7 +- .../DumpRSAccessionsStepConfiguration.java | 9 +- ...cessionReleaseFromDBStepConfiguration.java | 7 +- .../release/parameters/InputParameters.java | 10 - .../src/main/resources/application.properties | 1 - .../AccessionedVariantMongoReaderTest.java | 2 +- .../io/contig/ContigMongoReaderTest.java | 2 +- .../DeprecatedVariantMongoReaderTest.java | 2 +- .../merged/MergedVariantMongoReaderTest.java | 2 +- ...ergedDeprecatedVariantMongoReaderTest.java | 2 +- .../batch/steps/DumpRSAccessionsTest.java | 8 +- .../steps/RSAccessionsReleaseFromDBTest.java | 8 +- .../BatchJobRepositoryTestConfiguration.java | 22 -- .../configuration/BatchTestConfiguration.java | 2 - .../configuration/MongoTestConfiguration.java | 105 --------- .../ReleaseFromDBTestConfiguration.java | 3 - .../src/test/resources/application.properties | 1 - .../dump-rs-accession-test.properties | 1 - ...cremental-release-pipeline-test.properties | 27 --- .../release-pipeline-test.properties | 28 --- eva-accession-ws/pom.xml | 6 + .../accession/ws/EvaAccessionApplication.java | 16 +- .../ws/rest/EvaControllerAdvice.java | 2 +- .../src/main/resources/application.properties | 5 + .../ClusteredVariantHistoryEndPointTest.java | 5 +- .../ClusteredVariantsRestControllerTest.java | 5 +- .../SubmittedVariantsRestControllerTest.java | 28 ++- .../ws/test/MongoTestConfiguration.java | 105 --------- .../resources/accession-ws-test.properties | 5 + .../test-data/contiguous_id_blocks_schema.sql | 14 ++ .../ebi/eva/remapping/source/Application.java | 18 +- ...portSubmittedVariantsJobConfiguration.java | 17 +- ...ortSubmittedVariantsStepConfiguration.java | 8 +- .../source/parameters/InputParameters.java | 10 - ...RemappingJobLauncherCommandLineRunner.java | 16 +- .../src/main/resources/application.properties | 9 - .../DbsnpSubmittedVariantMongoReaderTest.java | 2 +- .../EvaSubmittedVariantMongoReaderTest.java | 2 +- ...SubmittedVariantsJobConfigurationTest.java | 6 +- ...ppingJobLauncherCommandLineRunnerTest.java | 6 +- ...cherCommandLineRunnerWithProjectsTest.java | 6 +- .../BatchJobRepositoryTestConfiguration.java | 22 -- .../configuration/BatchTestConfiguration.java | 6 +- .../configuration/MongoTestConfiguration.java | 105 --------- .../src/test/resources/application.properties | 1 - .../test/resources/with-projects.properties | 1 - .../ebi/eva/remapping/ingest/Application.java | 18 +- ...mappedVariantsFromVcfJobConfiguration.java | 4 - ...ngestRemappedFromVcfStepConfiguration.java | 6 +- ...oreRemappingMetadataStepConfiguration.java | 9 +- .../ingest/parameters/InputParameters.java | 10 - .../IngestRemappedCommandLineRunner.java | 16 +- .../RemappedSubmittedVariantsWriterTest.java | 2 +- ...edVariantsFromVcfJobConfigurationTest.java | 6 +- ...tRemappedFromVcfStepConfigurationTest.java | 6 +- ...ppedFromVcfStepDbsnpConfigurationTest.java | 6 +- .../BatchJobRepositoryTestConfiguration.java | 22 -- .../configuration/BatchTestConfiguration.java | 6 +- .../configuration/MongoTestConfiguration.java | 102 --------- pom.xml | 11 - 202 files changed, 1207 insertions(+), 2421 deletions(-) delete mode 100644 eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/test/configuration/BatchJobRepositoryTestConfiguration.java delete mode 100644 eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/test/configuration/MongoTestConfiguration.java create mode 100644 eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/configuration/ContiguousIdBlocksDataSourceConfiguration.java create mode 100644 eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/configuration/InMemoryBatchConfiguration.java create mode 100644 eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/configuration/NoOpJobExplorer.java delete mode 100644 eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/runner/CommandLineRunnerUtils.java delete mode 100644 eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/test/configuration/JPATestConfiguration.java create mode 100644 eva-accession-core/src/test/resources/test-data/contiguous_id_blocks_schema.sql delete mode 100644 eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/test/configuration/BatchJobRepositoryTestConfiguration.java delete mode 100644 eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/test/configuration/MongoTestConfiguration.java delete mode 100644 eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/test/BatchJobRepositoryTestConfiguration.java delete mode 100644 eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/test/MongoTestConfiguration.java delete mode 100644 eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/runner/RestartFailedJobTest.java delete mode 100644 eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/test/BatchJobRepositoryTestConfiguration.java delete mode 100644 eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/test/MongoTestConfiguration.java delete mode 100644 eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/test/configuration/BatchJobRepositoryTestConfiguration.java delete mode 100644 eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/test/configuration/MongoTestConfiguration.java delete mode 100644 eva-accession-release/src/test/resources/incremental-release-pipeline-test.properties delete mode 100644 eva-accession-release/src/test/resources/release-pipeline-test.properties delete mode 100644 eva-accession-ws/src/test/java/uk/ac/ebi/eva/accession/ws/test/MongoTestConfiguration.java create mode 100644 eva-accession-ws/src/test/resources/test-data/contiguous_id_blocks_schema.sql delete mode 100644 eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/test/configuration/BatchJobRepositoryTestConfiguration.java delete mode 100644 eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/test/configuration/MongoTestConfiguration.java delete mode 100644 eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/test/configuration/BatchJobRepositoryTestConfiguration.java delete mode 100644 eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/test/configuration/MongoTestConfiguration.java diff --git a/eva-accession-clustering/pom.xml b/eva-accession-clustering/pom.xml index 53ac31e80..d699bec1f 100644 --- a/eva-accession-clustering/pom.xml +++ b/eva-accession-clustering/pom.xml @@ -36,6 +36,11 @@ gson + + org.postgresql + postgresql + + com.thoughtworks.xstream xstream @@ -61,6 +66,12 @@ test + + org.hsqldb + hsqldb + test + + org.springframework.boot spring-boot-testcontainers diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/Application.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/Application.java index 0f009be40..9e135c0f8 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/Application.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/Application.java @@ -15,13 +15,26 @@ */ package uk.ac.ebi.eva.accession.clustering; -import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.autoconfigure.batch.BatchAutoConfiguration; +import org.springframework.boot.autoconfigure.data.mongo.MongoDataAutoConfiguration; +import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration; +import org.springframework.boot.autoconfigure.jdbc.DataSourceTransactionManagerAutoConfiguration; +import org.springframework.boot.autoconfigure.orm.jpa.HibernateJpaAutoConfiguration; import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.context.annotation.Import; +import uk.ac.ebi.eva.accession.core.configuration.ContiguousIdBlocksDataSourceConfiguration; +import uk.ac.ebi.eva.accession.core.configuration.InMemoryBatchConfiguration; -@EnableBatchProcessing -@SpringBootApplication +@SpringBootApplication(exclude = { + MongoDataAutoConfiguration.class, + DataSourceAutoConfiguration.class, + DataSourceTransactionManagerAutoConfiguration.class, + HibernateJpaAutoConfiguration.class, + BatchAutoConfiguration.class +}) +@Import({InMemoryBatchConfiguration.class, ContiguousIdBlocksDataSourceConfiguration.class}) public class Application { public static void main(String[] args) { diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/BackPropagateRSJobConfiguration.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/BackPropagateRSJobConfiguration.java index ec9f999b8..620d4c892 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/BackPropagateRSJobConfiguration.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/BackPropagateRSJobConfiguration.java @@ -17,9 +17,7 @@ import org.springframework.batch.core.Job; import org.springframework.batch.core.Step; -import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.batch.core.job.builder.JobBuilder; -import org.springframework.batch.core.launch.support.RunIdIncrementer; import org.springframework.batch.core.repository.JobRepository; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; @@ -31,7 +29,6 @@ import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.BACK_PROPAGATE_SPLIT_OR_MERGED_RS_STEP; @Configuration -@EnableBatchProcessing public class BackPropagateRSJobConfiguration { // Deal with back-propagation of new RS, that were assigned to SS in the remapped assembly, to the original assembly @Bean(BACK_PROPAGATE_NEW_RS_JOB) @@ -40,7 +37,6 @@ public Job backPropagateNewRSJob( @Qualifier(BACK_PROPAGATE_NEW_RS_STEP) Step backPropagateNewRSStep, JobRepository jobRepository) { return new JobBuilder(BACK_PROPAGATE_NEW_RS_JOB, jobRepository) - .incrementer(new RunIdIncrementer()) .start(backPropagateNewRSStep) .build(); } @@ -53,7 +49,6 @@ public Job backPropagateSplitOrMergedRSJob( @Qualifier(BACK_PROPAGATE_SPLIT_OR_MERGED_RS_STEP) Step backPropagateSplitMergedRSStep, JobRepository jobRepository) { return new JobBuilder(BACK_PROPAGATE_SPLIT_OR_MERGED_RS_JOB, jobRepository) - .incrementer(new RunIdIncrementer()) .start(backPropagateSplitMergedRSStep) .build(); } diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/ClusterUnclusteredVariantsJobConfiguration.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/ClusterUnclusteredVariantsJobConfiguration.java index 3740f1fd2..d46e3c84d 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/ClusterUnclusteredVariantsJobConfiguration.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/ClusterUnclusteredVariantsJobConfiguration.java @@ -18,9 +18,7 @@ import org.springframework.batch.core.Job; import org.springframework.batch.core.JobExecutionListener; import org.springframework.batch.core.Step; -import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.batch.core.job.builder.JobBuilder; -import org.springframework.batch.core.launch.support.RunIdIncrementer; import org.springframework.batch.core.repository.JobRepository; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; @@ -35,7 +33,6 @@ import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.PROCESS_RS_SPLIT_CANDIDATES_STEP; @Configuration -@EnableBatchProcessing public class ClusterUnclusteredVariantsJobConfiguration { // Should be run after split or merge candidates have been processed in the step @see PROCESS_REMAPPED_VARIANTS_WITH_RS_JOB // In this step, proceed to cluster as-yet unclustered variants in a given assembly @@ -50,7 +47,6 @@ public Job clusteringFromMongoJob( @Qualifier(JOB_EXECUTION_LISTENER) JobExecutionListener jobExecutionListener, JobRepository jobRepository) { return new JobBuilder(CLUSTER_UNCLUSTERED_VARIANTS_JOB, jobRepository) - .incrementer(new RunIdIncrementer()) .start(processRSMergeCandidatesStep) .next(processRSSplitCandidatesStep) .next(clearRSMergeAndSplitCandidatesStep) diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/ClusteringFromMongoJobConfiguration.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/ClusteringFromMongoJobConfiguration.java index e4c27393d..7f9686d8b 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/ClusteringFromMongoJobConfiguration.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/ClusteringFromMongoJobConfiguration.java @@ -21,13 +21,11 @@ import org.springframework.batch.core.JobExecutionListener; import org.springframework.batch.core.Step; import org.springframework.batch.core.StepExecution; -import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.batch.core.job.builder.FlowBuilder; import org.springframework.batch.core.job.builder.JobBuilder; import org.springframework.batch.core.job.flow.FlowExecutionStatus; import org.springframework.batch.core.job.flow.JobExecutionDecider; import org.springframework.batch.core.job.flow.support.SimpleFlow; -import org.springframework.batch.core.launch.support.RunIdIncrementer; import org.springframework.batch.core.repository.JobRepository; import org.springframework.batch.core.step.builder.StepBuilder; import org.springframework.batch.repeat.RepeatStatus; @@ -48,9 +46,9 @@ import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.JOB_EXECUTION_LISTENER; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.PROCESS_RS_MERGE_CANDIDATES_STEP; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.PROCESS_RS_SPLIT_CANDIDATES_STEP; +import static uk.ac.ebi.eva.accession.core.configuration.InMemoryBatchConfiguration.BATCH_TRANSACTION_MANAGER; @Configuration -@EnableBatchProcessing public class ClusteringFromMongoJobConfiguration { private JobExecutionDecider isRemappedAssemblyPresent(InputParameters inputParameters) { return new JobExecutionDecider() { @@ -63,7 +61,8 @@ public FlowExecutionStatus decide(@NonNull JobExecution jobExecution, StepExecut }; } - private Step dummyStep(JobRepository jobRepository, PlatformTransactionManager transactionManager) { + private Step dummyStep(JobRepository jobRepository, + @Qualifier(BATCH_TRANSACTION_MANAGER) PlatformTransactionManager transactionManager) { return new StepBuilder("step_" + "dummyStep", jobRepository) .tasklet((stepContribution, chunkContext) -> RepeatStatus.FINISHED, transactionManager) @@ -83,12 +82,12 @@ public Job clusteringFromMongoJob(@Qualifier(CLUSTERING_CLUSTERED_VARIANTS_FROM_ @Qualifier(BACK_PROPAGATE_SPLIT_OR_MERGED_RS_STEP) Step backPropagateSplitMergedRSStep, @Qualifier(JOB_EXECUTION_LISTENER) JobExecutionListener jobExecutionListener, - JobRepository jobRepository, PlatformTransactionManager transactionManager, + JobRepository jobRepository, + @Qualifier(BATCH_TRANSACTION_MANAGER) PlatformTransactionManager transactionManager, InputParameters inputParameters) { JobExecutionDecider jobExecutionDecider = isRemappedAssemblyPresent(inputParameters); Step dummyStep = dummyStep(jobRepository, transactionManager); return new JobBuilder(CLUSTERING_FROM_MONGO_JOB, jobRepository) - .incrementer(new RunIdIncrementer()) //We need the dummy step here because Spring won't conditionally start the first step .start(dummyStep) .listener(jobExecutionListener) diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/ProcessRemappedVariantsWithRSJobConfiguration.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/ProcessRemappedVariantsWithRSJobConfiguration.java index 6a9092b39..a0081fb95 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/ProcessRemappedVariantsWithRSJobConfiguration.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/ProcessRemappedVariantsWithRSJobConfiguration.java @@ -17,9 +17,7 @@ import org.springframework.batch.core.Job; import org.springframework.batch.core.Step; -import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.batch.core.job.builder.JobBuilder; -import org.springframework.batch.core.launch.support.RunIdIncrementer; import org.springframework.batch.core.repository.JobRepository; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; @@ -29,7 +27,6 @@ import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.PROCESS_REMAPPED_VARIANTS_WITH_RS_JOB; @Configuration -@EnableBatchProcessing public class ProcessRemappedVariantsWithRSJobConfiguration { // Deal with remapped variants with an existing RS and create split or merge candidates as needed // Can be parallelized across multiple species @@ -38,7 +35,6 @@ public Job processRemappedVariantsWithRSJob( @Qualifier(CLUSTERING_CLUSTERED_VARIANTS_FROM_MONGO_STEP) Step clusteringClusteredVariantsFromMongoStep, JobRepository jobRepository) { return new JobBuilder(PROCESS_REMAPPED_VARIANTS_WITH_RS_JOB, jobRepository) - .incrementer(new RunIdIncrementer()) .start(clusteringClusteredVariantsFromMongoStep) .build(); } diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/RSAccessionRecoveryJobConfiguration.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/RSAccessionRecoveryJobConfiguration.java index 7070c9dbc..9755305ae 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/RSAccessionRecoveryJobConfiguration.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/RSAccessionRecoveryJobConfiguration.java @@ -3,9 +3,7 @@ import org.springframework.batch.core.Job; import org.springframework.batch.core.JobExecutionListener; import org.springframework.batch.core.Step; -import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.batch.core.job.builder.JobBuilder; -import org.springframework.batch.core.launch.support.RunIdIncrementer; import org.springframework.batch.core.repository.JobRepository; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; @@ -17,7 +15,6 @@ import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.RS_ACCESSION_RECOVERY_STEP; @Configuration -@EnableBatchProcessing public class RSAccessionRecoveryJobConfiguration { @Autowired @@ -31,7 +28,6 @@ public class RSAccessionRecoveryJobConfiguration { @Bean(RS_ACCESSION_RECOVERY_JOB) public Job createMonotonicAccessionRecoveryAgentCategoryRSJob(JobRepository jobRepository) { return new JobBuilder(RS_ACCESSION_RECOVERY_JOB, jobRepository) - .incrementer(new RunIdIncrementer()) .start(monotonicAccessionRecoveryAgentCategoryRSStep) .listener(monotonicAccessionRecoveryAgentCategoryRSJobListener) .build(); diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/StudyClusteringJobConfiguration.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/StudyClusteringJobConfiguration.java index 36344d7e6..e15dc925f 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/StudyClusteringJobConfiguration.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/StudyClusteringJobConfiguration.java @@ -18,9 +18,7 @@ import org.springframework.batch.core.Job; import org.springframework.batch.core.JobExecutionListener; import org.springframework.batch.core.Step; -import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.batch.core.job.builder.JobBuilder; -import org.springframework.batch.core.launch.support.RunIdIncrementer; import org.springframework.batch.core.repository.JobRepository; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; @@ -32,7 +30,6 @@ import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.STUDY_CLUSTERING_STEP; @Configuration -@EnableBatchProcessing public class StudyClusteringJobConfiguration { @Bean(STUDY_CLUSTERING_JOB) @@ -41,7 +38,6 @@ public Job studyClusteringJob(@Qualifier(STUDY_CLUSTERING_STEP) Step clusteringS @Qualifier(JOB_EXECUTION_LISTENER) JobExecutionListener jobExecutionListener, JobRepository jobRepository) { return new JobBuilder(STUDY_CLUSTERING_JOB, jobRepository) - .incrementer(new RunIdIncrementer()) .start(clusteringStep) .next(accessioningShutdownStep) .listener(jobExecutionListener) diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/qc/ClusteringQCJobConfiguration.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/qc/ClusteringQCJobConfiguration.java index 69717ddfd..523f0e98d 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/qc/ClusteringQCJobConfiguration.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/qc/ClusteringQCJobConfiguration.java @@ -18,9 +18,7 @@ import org.springframework.batch.core.Job; import org.springframework.batch.core.Step; import org.springframework.batch.core.StepExecutionListener; -import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.batch.core.job.builder.JobBuilder; -import org.springframework.batch.core.launch.support.RunIdIncrementer; import org.springframework.batch.core.repository.JobRepository; import org.springframework.batch.core.step.builder.StepBuilder; import org.springframework.batch.item.ItemProcessor; @@ -44,9 +42,9 @@ import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.CLUSTERED_CLUSTERING_WRITER; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.PROGRESS_LISTENER; +import static uk.ac.ebi.eva.accession.core.configuration.InMemoryBatchConfiguration.BATCH_TRANSACTION_MANAGER; @Configuration -@EnableBatchProcessing // Since this is a QC Job, the configuration is intentionally lightweight and all collected in one-place // to reduce overhead public class ClusteringQCJobConfiguration { @@ -97,7 +95,8 @@ public Step reportUnclusteredSSAndPendingMergeSplitStep( @Qualifier(PENDING_MERGE_AND_SPLIT_REPORTER) ItemWriter pendingMergeSplitReporter, @Qualifier(PROGRESS_LISTENER) StepExecutionListener progressListener, - JobRepository jobRepository, PlatformTransactionManager transactionManager, + JobRepository jobRepository, + @Qualifier(BATCH_TRANSACTION_MANAGER) PlatformTransactionManager transactionManager, SimpleCompletionPolicy chunkSizeCompletionPolicy) { return new StepBuilder(REPORT_UNCLUSTERED_SS_AND_PENDING_MERGES_AND_SPLITS_STEP, jobRepository) .chunk(chunkSizeCompletionPolicy, transactionManager) @@ -115,7 +114,8 @@ public Step reportExtraneousRSStep( @Qualifier(EXTRANEOUS_RS_REPORTER) ItemWriter extraneousRSReporter, @Qualifier(PROGRESS_LISTENER) StepExecutionListener progressListener, - JobRepository jobRepository, PlatformTransactionManager transactionManager, + JobRepository jobRepository, + @Qualifier(BATCH_TRANSACTION_MANAGER) PlatformTransactionManager transactionManager, SimpleCompletionPolicy chunkSizeCompletionPolicy) { return new StepBuilder(REPORT_EXTRANEOUS_RS_STEP, jobRepository) .chunk(chunkSizeCompletionPolicy, transactionManager) @@ -133,7 +133,6 @@ public Job ClusteringQCJob( Step reportExtraneousRSStep, JobRepository jobRepository) { return new JobBuilder(CLUSTERING_QC_JOB, jobRepository) - .incrementer(new RunIdIncrementer()) .start(reportUnclusteredSSAndPendingMergeSplitStep) .next(reportExtraneousRSStep) .build(); diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/qc/DuplicateRSAccQCJobConfiguration.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/qc/DuplicateRSAccQCJobConfiguration.java index 48294ee59..840132874 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/qc/DuplicateRSAccQCJobConfiguration.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/qc/DuplicateRSAccQCJobConfiguration.java @@ -2,9 +2,7 @@ import org.springframework.batch.core.Job; import org.springframework.batch.core.Step; -import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.batch.core.job.builder.JobBuilder; -import org.springframework.batch.core.launch.support.RunIdIncrementer; import org.springframework.batch.core.repository.JobRepository; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; @@ -15,7 +13,6 @@ import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.DUPLICATE_RS_ACC_QC_STEP; @Configuration -@EnableBatchProcessing public class DuplicateRSAccQCJobConfiguration { @Autowired @@ -25,7 +22,6 @@ public class DuplicateRSAccQCJobConfiguration { @Bean(DUPLICATE_RS_ACC_QC_JOB) public Job duplicateRSAccQCJob(JobRepository jobRepository) { return new JobBuilder(DUPLICATE_RS_ACC_QC_JOB, jobRepository) - .incrementer(new RunIdIncrementer()) .start(duplicateRSAccQCStep) .build(); } diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/qc/NewClusteredVariantsQCJobConfiguration.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/qc/NewClusteredVariantsQCJobConfiguration.java index beddc0eaa..a41f8aa8d 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/qc/NewClusteredVariantsQCJobConfiguration.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/qc/NewClusteredVariantsQCJobConfiguration.java @@ -17,9 +17,7 @@ import org.springframework.batch.core.Job; import org.springframework.batch.core.Step; -import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.batch.core.job.builder.JobBuilder; -import org.springframework.batch.core.launch.support.RunIdIncrementer; import org.springframework.batch.core.repository.JobRepository; import org.springframework.batch.core.step.builder.StepBuilder; import org.springframework.batch.item.ItemStreamReader; @@ -38,8 +36,9 @@ import java.io.File; +import static uk.ac.ebi.eva.accession.core.configuration.InMemoryBatchConfiguration.BATCH_TRANSACTION_MANAGER; + @Configuration -@EnableBatchProcessing // Since this is a QC Job, the configuration is intentionally lightweight and all collected in one-place // to reduce overhead public class NewClusteredVariantsQCJobConfiguration { @@ -80,7 +79,8 @@ public MissingCveReporter missingCveReporter(MongoTemplate mongoTemplate) { public Step reportMissingCveStep( @Qualifier(RS_REPORT_READER) ItemStreamReader rsReportReader, @Qualifier(MISSING_CVE_REPORTER) ItemWriter missingCveReporter, - JobRepository jobRepository, PlatformTransactionManager transactionManager, + JobRepository jobRepository, + @Qualifier(BATCH_TRANSACTION_MANAGER) PlatformTransactionManager transactionManager, SimpleCompletionPolicy chunkSizeCompletionPolicy) { return new StepBuilder(REPORT_MISSING_CVE_STEP, jobRepository) .chunk(chunkSizeCompletionPolicy, transactionManager) @@ -93,7 +93,6 @@ public Step reportMissingCveStep( public Job ClusteringQCJob(@Qualifier(REPORT_MISSING_CVE_STEP) Step reportMissingCveStep, JobRepository jobRepository) { return new JobBuilder(NEW_CLUSTERED_VARIANTS_QC_JOB, jobRepository) - .incrementer(new RunIdIncrementer()) .start(reportMissingCveStep) .build(); } diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/steps/AccessioningShutdownStepConfiguration.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/steps/AccessioningShutdownStepConfiguration.java index 766cf292b..27269eda8 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/steps/AccessioningShutdownStepConfiguration.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/steps/AccessioningShutdownStepConfiguration.java @@ -1,10 +1,10 @@ package uk.ac.ebi.eva.accession.clustering.configuration.batch.steps; import org.springframework.batch.core.Step; -import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.batch.core.repository.JobRepository; import org.springframework.batch.core.step.builder.StepBuilder; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.transaction.PlatformTransactionManager; @@ -12,9 +12,9 @@ import uk.ac.ebi.eva.accession.core.service.nonhuman.SubmittedVariantAccessioningService; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.ACCESSIONING_SHUTDOWN_STEP; +import static uk.ac.ebi.eva.accession.core.configuration.InMemoryBatchConfiguration.BATCH_TRANSACTION_MANAGER; @Configuration -@EnableBatchProcessing public class AccessioningShutdownStepConfiguration { @Autowired private SubmittedVariantAccessioningService submittedVariantAccessioningService; @@ -23,7 +23,8 @@ public class AccessioningShutdownStepConfiguration { private ClusteredVariantAccessioningService clusteredVariantAccessioningService; @Bean(ACCESSIONING_SHUTDOWN_STEP) - public Step accessioningShutDownStep(JobRepository jobRepository, PlatformTransactionManager transactionManager) { + public Step accessioningShutDownStep(JobRepository jobRepository, + @Qualifier(BATCH_TRANSACTION_MANAGER) PlatformTransactionManager transactionManager) { return new StepBuilder(ACCESSIONING_SHUTDOWN_STEP, jobRepository) .tasklet((contribution, chunkContext) -> { submittedVariantAccessioningService.shutDownAccessionGenerator(); diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/steps/ClusteringFromMongoStepConfiguration.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/steps/ClusteringFromMongoStepConfiguration.java index 41417be99..2fb319007 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/steps/ClusteringFromMongoStepConfiguration.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/steps/ClusteringFromMongoStepConfiguration.java @@ -18,7 +18,6 @@ import org.springframework.batch.core.Step; import org.springframework.batch.core.StepExecutionListener; -import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.batch.core.repository.JobRepository; import org.springframework.batch.core.step.builder.StepBuilder; import org.springframework.batch.core.step.tasklet.TaskletStep; @@ -61,9 +60,9 @@ import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.STUDY_CLUSTERING_STEP; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.TARGET_SS_READER_FOR_NEW_BACKPROP_RS; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.TARGET_SS_READER_FOR_SPLIT_OR_MERGED_BACKPROP_RS; +import static uk.ac.ebi.eva.accession.core.configuration.InMemoryBatchConfiguration.BATCH_TRANSACTION_MANAGER; @Configuration -@EnableBatchProcessing public class ClusteringFromMongoStepConfiguration { @Bean(CLUSTERING_CLUSTERED_VARIANTS_FROM_MONGO_STEP) @@ -72,7 +71,8 @@ public Step clusteringClusteredVariantStepMongoReader( @Qualifier(CLUSTERED_CLUSTERING_WRITER) ItemWriter submittedVariantWriter, @Qualifier(PROGRESS_LISTENER) StepExecutionListener progressListener, @Qualifier(CLUSTERED_CLUSTERING_WRITER_JOB_EXECUTION_SETTER) StepExecutionListener clusteredClusteringWriterJobExecutionSetter, - JobRepository jobRepository, PlatformTransactionManager transactionManager, + JobRepository jobRepository, + @Qualifier(BATCH_TRANSACTION_MANAGER) PlatformTransactionManager transactionManager, SimpleCompletionPolicy chunkSizeCompletionPolicy) { TaskletStep step = new StepBuilder(CLUSTERING_CLUSTERED_VARIANTS_FROM_MONGO_STEP, jobRepository) .chunk(chunkSizeCompletionPolicy, transactionManager) @@ -90,7 +90,8 @@ public Step processRSMergeCandidatesStep( ItemReader rsMergeCandidatesReader, @Qualifier(RS_MERGE_WRITER) ItemWriter rsMergeWriter, @Qualifier(PROGRESS_LISTENER) StepExecutionListener progressListener, - JobRepository jobRepository, PlatformTransactionManager transactionManager, + JobRepository jobRepository, + @Qualifier(BATCH_TRANSACTION_MANAGER) PlatformTransactionManager transactionManager, SimpleCompletionPolicy chunkSizeCompletionPolicy) { TaskletStep step = new StepBuilder(PROCESS_RS_MERGE_CANDIDATES_STEP, jobRepository) .chunk(chunkSizeCompletionPolicy, transactionManager) @@ -108,7 +109,8 @@ public Step processRSSplitCandidatesStep( @Qualifier(RS_SPLIT_WRITER) ItemWriter rsSplitWriter, @Qualifier(PROGRESS_LISTENER) StepExecutionListener progressListener, @Qualifier(RS_SPLIT_WRITER_JOB_EXECUTION_SETTER) StepExecutionListener rsSplitWriterJobExecutionSetter, - JobRepository jobRepository, PlatformTransactionManager transactionManager, + JobRepository jobRepository, + @Qualifier(BATCH_TRANSACTION_MANAGER) PlatformTransactionManager transactionManager, SimpleCompletionPolicy chunkSizeCompletionPolicy) { TaskletStep step = new StepBuilder(PROCESS_RS_SPLIT_CANDIDATES_STEP, jobRepository) .chunk(chunkSizeCompletionPolicy, transactionManager) @@ -123,7 +125,8 @@ public Step processRSSplitCandidatesStep( @Bean(CLEAR_RS_MERGE_AND_SPLIT_CANDIDATES_STEP) public Step clearRSMergeAndSplitCandidatesStep( @Qualifier(CLEAR_RS_MERGE_AND_SPLIT_CANDIDATES) ItemWriter clearRSMergeAndSplitCandidates, - JobRepository jobRepository, PlatformTransactionManager transactionManager, + JobRepository jobRepository, + @Qualifier(BATCH_TRANSACTION_MANAGER) PlatformTransactionManager transactionManager, SimpleCompletionPolicy chunkSizeCompletionPolicy) { TaskletStep step = new StepBuilder(CLEAR_RS_MERGE_AND_SPLIT_CANDIDATES_STEP, jobRepository) .chunk(chunkSizeCompletionPolicy, transactionManager) @@ -155,7 +158,8 @@ public Step clusteringNonClusteredVariantStepMongoReader( @Qualifier(NON_CLUSTERED_CLUSTERING_WRITER) ItemWriter submittedVariantWriter, @Qualifier(PROGRESS_LISTENER) StepExecutionListener progressListener, @Qualifier(NON_CLUSTERED_CLUSTERING_WRITER_JOB_EXECUTION_SETTER) StepExecutionListener nonClusteredClusteringWriterJobExecutionSetter, - JobRepository jobRepository, PlatformTransactionManager transactionManager, + JobRepository jobRepository, + @Qualifier(BATCH_TRANSACTION_MANAGER) PlatformTransactionManager transactionManager, SimpleCompletionPolicy chunkSizeCompletionPolicy) { TaskletStep step = new StepBuilder(CLUSTERING_NON_CLUSTERED_VARIANTS_FROM_MONGO_STEP, jobRepository) .chunk(chunkSizeCompletionPolicy, transactionManager) @@ -173,7 +177,8 @@ public Step backPropagateNewRSStep( ItemStreamReader backPropagatedNewRSTargetReader, @Qualifier(BACK_PROPAGATED_RS_WRITER) ItemWriter backPropagatedRSWriter, @Qualifier(PROGRESS_LISTENER) StepExecutionListener progressListener, - JobRepository jobRepository, PlatformTransactionManager transactionManager, + JobRepository jobRepository, + @Qualifier(BATCH_TRANSACTION_MANAGER) PlatformTransactionManager transactionManager, SimpleCompletionPolicy chunkSizeCompletionPolicy) { TaskletStep step = new StepBuilder(BACK_PROPAGATE_NEW_RS_STEP, jobRepository) .chunk(chunkSizeCompletionPolicy, transactionManager) @@ -190,7 +195,8 @@ public Step backPropagateSplitAndMergedRSStep( ItemStreamReader> splitOrMergedRSReader, @Qualifier(BACK_PROPAGATED_RS_WRITER) ItemWriter backPropagatedRSWriter, @Qualifier(PROGRESS_LISTENER) StepExecutionListener progressListener, - JobRepository jobRepository, PlatformTransactionManager transactionManager, + JobRepository jobRepository, + @Qualifier(BATCH_TRANSACTION_MANAGER) PlatformTransactionManager transactionManager, SimpleCompletionPolicy chunkSizeCompletionPolicy) { TaskletStep step = new StepBuilder(BACK_PROPAGATE_SPLIT_OR_MERGED_RS_STEP, jobRepository) ., List>chunk(chunkSizeCompletionPolicy, transactionManager) @@ -208,7 +214,8 @@ public Step studyClusteringStep( @Qualifier(NON_CLUSTERED_CLUSTERING_WRITER) ItemWriter submittedVariantWriter, @Qualifier(PROGRESS_LISTENER) StepExecutionListener progressListener, @Qualifier(NON_CLUSTERED_CLUSTERING_WRITER_JOB_EXECUTION_SETTER) StepExecutionListener nonClusteredClusteringWriterJobExecutionSetter, - JobRepository jobRepository, PlatformTransactionManager transactionManager, + JobRepository jobRepository, + @Qualifier(BATCH_TRANSACTION_MANAGER) PlatformTransactionManager transactionManager, SimpleCompletionPolicy chunkSizeCompletionPolicy) { TaskletStep step = new StepBuilder(STUDY_CLUSTERING_STEP, jobRepository) .chunk(chunkSizeCompletionPolicy, transactionManager) diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/steps/RSAccessionRecoveryStepConfiguration.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/steps/RSAccessionRecoveryStepConfiguration.java index f484eda83..260a8aa2e 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/steps/RSAccessionRecoveryStepConfiguration.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/steps/RSAccessionRecoveryStepConfiguration.java @@ -1,7 +1,6 @@ package uk.ac.ebi.eva.accession.clustering.configuration.batch.steps; import org.springframework.batch.core.Step; -import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.batch.core.repository.JobRepository; import org.springframework.batch.core.step.builder.StepBuilder; import org.springframework.beans.factory.annotation.Autowired; @@ -13,9 +12,9 @@ import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.RS_ACCESSION_RECOVERY_SERVICE; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.RS_ACCESSION_RECOVERY_STEP; +import static uk.ac.ebi.eva.accession.core.configuration.InMemoryBatchConfiguration.BATCH_TRANSACTION_MANAGER; @Configuration -@EnableBatchProcessing public class RSAccessionRecoveryStepConfiguration { @Autowired @Qualifier(RS_ACCESSION_RECOVERY_SERVICE) @@ -23,6 +22,7 @@ public class RSAccessionRecoveryStepConfiguration { @Bean(RS_ACCESSION_RECOVERY_STEP) public Step monotonicAccessionRecoveryAgentCategoryRSStep(JobRepository jobRepository, + @Qualifier(BATCH_TRANSACTION_MANAGER) PlatformTransactionManager transactionManager) { return new StepBuilder(RS_ACCESSION_RECOVERY_STEP, jobRepository) .tasklet((contribution, chunkContext) -> { diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/steps/qc/DuplicateRSAccQCStepConfiguration.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/steps/qc/DuplicateRSAccQCStepConfiguration.java index 2c0afce12..ca8a0a73b 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/steps/qc/DuplicateRSAccQCStepConfiguration.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/steps/qc/DuplicateRSAccQCStepConfiguration.java @@ -1,7 +1,6 @@ package uk.ac.ebi.eva.accession.clustering.configuration.batch.steps.qc; import org.springframework.batch.core.Step; -import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.batch.core.repository.JobRepository; import org.springframework.batch.core.step.builder.StepBuilder; import org.springframework.batch.core.step.tasklet.TaskletStep; @@ -21,9 +20,9 @@ import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.DUPLICATE_RS_ACC_QC_PROCESSOR; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.DUPLICATE_RS_ACC_QC_STEP; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.DUPLICATE_RS_ACC_QC_WRITER; +import static uk.ac.ebi.eva.accession.core.configuration.InMemoryBatchConfiguration.BATCH_TRANSACTION_MANAGER; @Configuration -@EnableBatchProcessing public class DuplicateRSAccQCStepConfiguration { @Autowired @@ -39,7 +38,8 @@ public class DuplicateRSAccQCStepConfiguration { private ItemWriter> duplicateRSAccQCWriter; @Bean(DUPLICATE_RS_ACC_QC_STEP) - public Step duplicateRSAccQCStep(JobRepository jobRepository, PlatformTransactionManager transactionManager) { + public Step duplicateRSAccQCStep(JobRepository jobRepository, + @Qualifier(BATCH_TRANSACTION_MANAGER) PlatformTransactionManager transactionManager) { TaskletStep step = new StepBuilder(DUPLICATE_RS_ACC_QC_STEP, jobRepository) // hardcoded the chunk size as 1, as the reader takes care of accumulating // and sending the chunk size (defined in properties file) elements to the processor diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/parameters/InputParameters.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/parameters/InputParameters.java index c2128afb8..1edf29816 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/parameters/InputParameters.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/parameters/InputParameters.java @@ -33,8 +33,6 @@ public class InputParameters { private int chunkSize; - private boolean forceRestart; - private boolean allowRetry; private String rsAccFile; @@ -73,14 +71,6 @@ public void setChunkSize(int chunkSize) { this.chunkSize = chunkSize; } - public boolean isForceRestart() { - return forceRestart; - } - - public void setForceRestart(boolean forceRestart) { - this.forceRestart = forceRestart; - } - public boolean isAllowRetry() { return allowRetry; } diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/runner/ClusteringCommandLineRunner.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/runner/ClusteringCommandLineRunner.java index 25e60ba7f..477b73613 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/runner/ClusteringCommandLineRunner.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/runner/ClusteringCommandLineRunner.java @@ -19,7 +19,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.batch.core.Job; -import org.springframework.batch.core.JobExecution; import org.springframework.batch.core.JobExecutionException; import org.springframework.batch.core.JobParameters; import org.springframework.batch.core.JobParametersInvalidException; @@ -37,10 +36,8 @@ import org.springframework.stereotype.Component; import org.springframework.util.PatternMatchUtils; import uk.ac.ebi.eva.accession.clustering.parameters.InputParameters; -import uk.ac.ebi.eva.accession.core.runner.CommandLineRunnerUtils; import uk.ac.ebi.eva.commons.batch.exception.NoJobToExecuteException; import uk.ac.ebi.eva.commons.batch.exception.NoParametersHaveBeenPassedException; -import uk.ac.ebi.eva.commons.batch.exception.NoPreviousJobExecutionException; import uk.ac.ebi.eva.commons.batch.exception.UnknownJobException; import uk.ac.ebi.eva.commons.batch.job.JobExecutionApplicationListener; import uk.ac.ebi.eva.commons.batch.job.JobStatusManager; @@ -113,17 +110,9 @@ public void run(String... args) throws JobExecutionException { JobStatusManager.checkIfJobNameHasBeenDefined(jobName); JobStatusManager.checkIfPropertiesHaveBeenProvided(jobParameters); - if (inputParameters.isForceRestart()) { - JobExecution previousJobExecution = CommandLineRunnerUtils.getLastJobExecution(jobName, jobExplorer, - jobParameters); - CommandLineRunnerUtils.markPreviousJobAsFailed(jobName, jobRepository, - previousJobExecution != null ? previousJobExecution.getJobParameters() : jobParameters); - } else { - jobParameters = CommandLineRunnerUtils.addRunIDToJobParameters(jobName, jobExplorer, jobParameters); - } launchJob(jobParameters); - } catch (NoJobToExecuteException | NoParametersHaveBeenPassedException | NoPreviousJobExecutionException - | UnknownJobException | JobParametersInvalidException | JobExecutionAlreadyRunningException e) { + } catch (NoJobToExecuteException | NoParametersHaveBeenPassedException | UnknownJobException + | JobParametersInvalidException | JobExecutionAlreadyRunningException e) { logger.error(e.getMessage()); logger.debug("Error trace", e); abnormalExit = true; diff --git a/eva-accession-clustering/src/main/resources/application.properties b/eva-accession-clustering/src/main/resources/application.properties index d63886aef..b90cd9152 100644 --- a/eva-accession-clustering/src/main/resources/application.properties +++ b/eva-accession-clustering/src/main/resources/application.properties @@ -36,12 +36,6 @@ spring.data.mongodb.password= spring.data.mongodb.authentication-database= mongodb.read-preference=secondaryPreferred -spring.datasource.driver-class-name=org.postgresql.Driver -spring.datasource.url= -spring.datasource.username= -spring.datasource.password= -spring.datasource.tomcat.max-active=3 - # Only to set up the database! # jmmut: These don't work at the moment, maybe because we use both Spring 2 and Spring 1 through variation commons # spring.jpa.generate-ddl=true diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/BackPropagatedRSWriterTest.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/BackPropagatedRSWriterTest.java index a255bf923..cd851dd37 100644 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/BackPropagatedRSWriterTest.java +++ b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/BackPropagatedRSWriterTest.java @@ -31,7 +31,7 @@ import uk.ac.ebi.ampt2d.commons.accession.core.models.AccessionWrapper; import uk.ac.ebi.ampt2d.commons.accession.hashing.SHA1HashingFunction; import uk.ac.ebi.eva.accession.clustering.test.configuration.BatchTestConfiguration; -import uk.ac.ebi.eva.accession.clustering.test.configuration.MongoTestConfiguration; +import uk.ac.ebi.eva.accession.core.configuration.ContiguousIdBlocksDataSourceConfiguration; import uk.ac.ebi.eva.accession.core.model.ClusteredVariant; import uk.ac.ebi.eva.accession.core.model.IClusteredVariant; import uk.ac.ebi.eva.accession.core.model.ISubmittedVariant; @@ -43,6 +43,7 @@ import uk.ac.ebi.eva.accession.core.service.nonhuman.SubmittedVariantAccessioningService; import uk.ac.ebi.eva.accession.core.summary.ClusteredVariantSummaryFunction; import uk.ac.ebi.eva.accession.core.summary.SubmittedVariantSummaryFunction; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.commons.core.models.VariantType; import uk.ac.ebi.eva.metrics.metric.MetricCompute; @@ -59,7 +60,8 @@ import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.CLUSTERED_CLUSTERING_WRITER; @ExtendWith(SpringExtension.class) -@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class,}) +@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class, + ContiguousIdBlocksDataSourceConfiguration.class}) @TestPropertySource("classpath:backpropagation-test.properties") public class BackPropagatedRSWriterTest extends MongoTestContainerHelper { private static final int TAXONOMY = 60711; diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/ClusteringMongoReaderTest.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/ClusteringMongoReaderTest.java index c5971393b..734e8445b 100644 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/ClusteringMongoReaderTest.java +++ b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/ClusteringMongoReaderTest.java @@ -26,9 +26,9 @@ import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit.jupiter.SpringExtension; -import uk.ac.ebi.eva.accession.clustering.test.configuration.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.MongoConfiguration; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.core.utils.MongoTestDataLoader; diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/RSMergeAndSplitCandidatesReaderTest.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/RSMergeAndSplitCandidatesReaderTest.java index 6d14878c2..2c9d8e018 100644 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/RSMergeAndSplitCandidatesReaderTest.java +++ b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/RSMergeAndSplitCandidatesReaderTest.java @@ -28,10 +28,11 @@ import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.eva.accession.clustering.configuration.batch.io.RSMergeAndSplitCandidatesReaderConfiguration; import uk.ac.ebi.eva.accession.clustering.test.configuration.BatchTestConfiguration; -import uk.ac.ebi.eva.accession.clustering.test.configuration.MongoTestConfiguration; +import uk.ac.ebi.eva.accession.core.configuration.ContiguousIdBlocksDataSourceConfiguration; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantInactiveEntity; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantOperationEntity; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.commons.mongodb.readers.MongoDbCursorItemReader; @@ -46,11 +47,9 @@ @ExtendWith(SpringExtension.class) @TestPropertySource("classpath:clustering-pipeline-test.properties") @ContextConfiguration(classes = {RSMergeAndSplitCandidatesReaderConfiguration.class, MongoTestConfiguration.class, - BatchTestConfiguration.class}) + BatchTestConfiguration.class, ContiguousIdBlocksDataSourceConfiguration.class}) public class RSMergeAndSplitCandidatesReaderTest extends MongoTestContainerHelper { - private static final String TEST_DB = "test-db"; - private static final String ASSEMBLY = "GCA_000000001.1"; private static final String SUBMITTED_VARIANT_OPERATION_COLLECTION = "submittedVariantOperationEntity"; diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/RSMergeWriterTest.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/RSMergeWriterTest.java index c18812865..77e724730 100644 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/RSMergeWriterTest.java +++ b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/RSMergeWriterTest.java @@ -40,7 +40,7 @@ import uk.ac.ebi.eva.accession.clustering.metric.ClusteringMetric; import uk.ac.ebi.eva.accession.clustering.test.DatabaseState; import uk.ac.ebi.eva.accession.clustering.test.configuration.BatchTestConfiguration; -import uk.ac.ebi.eva.accession.clustering.test.configuration.MongoTestConfiguration; +import uk.ac.ebi.eva.accession.core.configuration.ContiguousIdBlocksDataSourceConfiguration; import uk.ac.ebi.eva.accession.core.model.ClusteredVariant; import uk.ac.ebi.eva.accession.core.model.IClusteredVariant; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpClusteredVariantEntity; @@ -54,6 +54,7 @@ import uk.ac.ebi.eva.accession.core.service.nonhuman.ClusteredVariantAccessioningService; import uk.ac.ebi.eva.accession.core.service.nonhuman.SubmittedVariantAccessioningService; import uk.ac.ebi.eva.accession.core.summary.ClusteredVariantSummaryFunction; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.commons.core.models.pipeline.Variant; import uk.ac.ebi.eva.commons.mongodb.readers.MongoDbCursorItemReader; @@ -79,7 +80,7 @@ @ExtendWith(SpringExtension.class) @TestPropertySource("classpath:merge-split-test.properties") @ContextConfiguration(classes = {RSMergeAndSplitCandidatesReaderConfiguration.class, RSMergeAndSplitWriterConfiguration.class, - MongoTestConfiguration.class, BatchTestConfiguration.class}) + MongoTestConfiguration.class, BatchTestConfiguration.class, ContiguousIdBlocksDataSourceConfiguration.class}) public class RSMergeWriterTest extends MongoTestContainerHelper { private static final String ASSEMBLY = "GCA_000000001.1"; diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/RSSplitWriterTest.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/RSSplitWriterTest.java index e2c326179..9c414ba4c 100644 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/RSSplitWriterTest.java +++ b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/RSSplitWriterTest.java @@ -25,7 +25,9 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.core.io.ClassPathResource; import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.jdbc.datasource.init.ResourceDatabasePopulator; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; @@ -37,7 +39,7 @@ import uk.ac.ebi.eva.accession.clustering.metric.ClusteringMetric; import uk.ac.ebi.eva.accession.clustering.test.DatabaseState; import uk.ac.ebi.eva.accession.clustering.test.configuration.BatchTestConfiguration; -import uk.ac.ebi.eva.accession.clustering.test.configuration.MongoTestConfiguration; +import uk.ac.ebi.eva.accession.core.configuration.ContiguousIdBlocksDataSourceConfiguration; import uk.ac.ebi.eva.accession.core.model.ISubmittedVariant; import uk.ac.ebi.eva.accession.core.model.SubmittedVariant; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpClusteredVariantEntity; @@ -48,12 +50,17 @@ import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantOperationEntity; import uk.ac.ebi.eva.accession.core.service.nonhuman.SubmittedVariantAccessioningService; import uk.ac.ebi.eva.accession.core.summary.SubmittedVariantSummaryFunction; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.metrics.metric.MetricCompute; +import javax.sql.DataSource; import java.io.File; import java.io.IOException; import java.nio.file.Files; +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -70,7 +77,9 @@ @ExtendWith(SpringExtension.class) @TestPropertySource("classpath:merge-split-test.properties") @ContextConfiguration(classes = {RSMergeAndSplitCandidatesReaderConfiguration.class, - RSMergeAndSplitWriterConfiguration.class, MongoTestConfiguration.class, BatchTestConfiguration.class}) + RSMergeAndSplitWriterConfiguration.class, MongoTestConfiguration.class, BatchTestConfiguration.class, + ContiguousIdBlocksDataSourceConfiguration.class}) +@DirtiesContext(classMode = DirtiesContext.ClassMode.BEFORE_EACH_TEST_METHOD) public class RSSplitWriterTest extends MongoTestContainerHelper { private static final String ASSEMBLY = "GCA_000000001.1"; @@ -92,6 +101,9 @@ public class RSSplitWriterTest extends MongoTestContainerHelper { @MockBean private JobExecution jobExecution; + @Autowired + private DataSource dataSource; + private SubmittedVariantEntity ss1, ss2, ss3, ss4, ss5, ss6; @Autowired @@ -101,8 +113,16 @@ public class RSSplitWriterTest extends MongoTestContainerHelper { private MetricCompute metricCompute; @BeforeEach - public void setUp() throws IOException { + public void setUp() throws IOException, SQLException { mongoTemplate.getDb().drop(); + try (Connection conn = dataSource.getConnection(); + Statement stmt = conn.createStatement()) { + stmt.execute("DROP TABLE IF EXISTS contiguous_id_blocks"); + } + ResourceDatabasePopulator populator = new ResourceDatabasePopulator(); + populator.addScript(new ClassPathResource("test-data/contiguous_id_blocks_schema.sql")); + populator.execute(dataSource); + metricCompute.clearCount(); Files.deleteIfExists(this.rsReportFile.toPath()); diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/SSSplitWriterTest.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/SSSplitWriterTest.java index cbe743db0..4f036f04f 100644 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/SSSplitWriterTest.java +++ b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/SSSplitWriterTest.java @@ -25,9 +25,12 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.core.io.ClassPathResource; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.http.HttpMethod; import org.springframework.http.HttpStatus; +import org.springframework.jdbc.datasource.init.ResourceDatabasePopulator; +import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit.jupiter.SpringExtension; @@ -39,7 +42,7 @@ import uk.ac.ebi.eva.accession.clustering.metric.ClusteringMetric; import uk.ac.ebi.eva.accession.clustering.test.DatabaseState; import uk.ac.ebi.eva.accession.clustering.test.configuration.BatchTestConfiguration; -import uk.ac.ebi.eva.accession.clustering.test.configuration.MongoTestConfiguration; +import uk.ac.ebi.eva.accession.core.configuration.ContiguousIdBlocksDataSourceConfiguration; import uk.ac.ebi.eva.accession.core.model.ISubmittedVariant; import uk.ac.ebi.eva.accession.core.model.SubmittedVariant; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpSubmittedVariantEntity; @@ -48,11 +51,15 @@ import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantOperationEntity; import uk.ac.ebi.eva.accession.core.service.nonhuman.SubmittedVariantAccessioningService; import uk.ac.ebi.eva.accession.core.summary.SubmittedVariantSummaryFunction; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.metrics.count.CountServiceParameters; import uk.ac.ebi.eva.metrics.metric.MetricCompute; +import javax.sql.DataSource; import java.net.URI; +import java.sql.Connection; +import java.sql.Statement; import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -73,7 +80,8 @@ @ExtendWith(SpringExtension.class) @TestPropertySource("classpath:merge-split-test.properties") @ContextConfiguration(classes = {SSSplitWriterConfiguration.class, MongoTestConfiguration.class, - BatchTestConfiguration.class}) + BatchTestConfiguration.class, ContiguousIdBlocksDataSourceConfiguration.class}) +@DirtiesContext(classMode = DirtiesContext.ClassMode.BEFORE_EACH_TEST_METHOD) public class SSSplitWriterTest extends MongoTestContainerHelper { private static final String ASSEMBLY = "GCA_000000001.1"; @@ -101,6 +109,9 @@ public class SSSplitWriterTest extends MongoTestContainerHelper { @MockBean private JobExecution jobExecution; + @Autowired + private DataSource dataSource; + private MockRestServiceServer mockServer; @Autowired @@ -112,6 +123,15 @@ public class SSSplitWriterTest extends MongoTestContainerHelper { @BeforeEach public void setUp() throws Exception { mongoTemplate.getDb().drop(); + + try (Connection conn = dataSource.getConnection(); + Statement stmt = conn.createStatement()) { + stmt.execute("DROP TABLE IF EXISTS contiguous_id_blocks"); + } + ResourceDatabasePopulator populator = new ResourceDatabasePopulator(); + populator.addScript(new ClassPathResource("test-data/contiguous_id_blocks_schema.sql")); + populator.execute(dataSource); + metricCompute.clearCount(); mockServer = MockRestServiceServer.createServer(restTemplate); diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/StudyClusteringMongoReaderTest.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/StudyClusteringMongoReaderTest.java index 74c30f7f2..9d55e7baa 100644 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/StudyClusteringMongoReaderTest.java +++ b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/StudyClusteringMongoReaderTest.java @@ -26,9 +26,9 @@ import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit.jupiter.SpringExtension; -import uk.ac.ebi.eva.accession.clustering.test.configuration.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.MongoConfiguration; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.core.utils.MongoTestDataLoader; diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/clustering_writer/IssueAccessionClusteringWriterTest.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/clustering_writer/IssueAccessionClusteringWriterTest.java index 39f560d3b..7c72f14f9 100644 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/clustering_writer/IssueAccessionClusteringWriterTest.java +++ b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/clustering_writer/IssueAccessionClusteringWriterTest.java @@ -29,12 +29,13 @@ import org.springframework.batch.item.ItemWriter; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.core.io.ClassPathResource; import org.springframework.core.io.ResourceLoader; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.data.mongodb.core.query.Criteria; import org.springframework.data.mongodb.core.query.Query; +import org.springframework.jdbc.datasource.init.ResourceDatabasePopulator; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; @@ -45,7 +46,7 @@ import uk.ac.ebi.eva.accession.clustering.batch.io.RSSplitWriter; import uk.ac.ebi.eva.accession.clustering.parameters.InputParameters; import uk.ac.ebi.eva.accession.clustering.test.configuration.BatchTestConfiguration; -import uk.ac.ebi.eva.accession.clustering.test.configuration.MongoTestConfiguration; +import uk.ac.ebi.eva.accession.core.configuration.ContiguousIdBlocksDataSourceConfiguration; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.ClusteredVariantAccessioningConfiguration; import uk.ac.ebi.eva.accession.core.model.ClusteredVariant; import uk.ac.ebi.eva.accession.core.model.IClusteredVariant; @@ -57,15 +58,20 @@ import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantOperationEntity; import uk.ac.ebi.eva.accession.core.summary.ClusteredVariantSummaryFunction; import uk.ac.ebi.eva.accession.core.summary.SubmittedVariantSummaryFunction; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.core.utils.MongoTestDataLoader; import uk.ac.ebi.eva.commons.core.models.VariantType; import uk.ac.ebi.eva.commons.mongodb.readers.MongoDbCursorItemReader; import uk.ac.ebi.eva.metrics.metric.MetricCompute; +import javax.sql.DataSource; import java.io.File; import java.io.IOException; import java.nio.file.Files; +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -92,8 +98,8 @@ * care of those scenarios. */ @ExtendWith(SpringExtension.class) -@EnableAutoConfiguration -@ContextConfiguration(classes = {ClusteredVariantAccessioningConfiguration.class, BatchTestConfiguration.class, MongoTestConfiguration.class}) +@ContextConfiguration(classes = {ClusteredVariantAccessioningConfiguration.class, BatchTestConfiguration.class, + MongoTestConfiguration.class, ContiguousIdBlocksDataSourceConfiguration.class}) @TestPropertySource("classpath:clustering-issuance-test.properties") public class IssueAccessionClusteringWriterTest extends MongoTestContainerHelper { @@ -154,13 +160,24 @@ public class IssueAccessionClusteringWriterTest extends MongoTestContainerHelper @MockBean private JobExecution jobExecution; + @Autowired + private DataSource dataSource; + private Function hashingFunction; private Function clusteredHashingFunction; @BeforeEach - public void setUp() throws IOException { + public void setUp() throws IOException, SQLException { mongoTemplate.getDb().drop(); + try (Connection conn = dataSource.getConnection(); + Statement stmt = conn.createStatement()) { + stmt.execute("DROP TABLE IF EXISTS contiguous_id_blocks"); + } + ResourceDatabasePopulator populator = new ResourceDatabasePopulator(); + populator.addScript(new ClassPathResource("test-data/contiguous_id_blocks_schema.sql")); + populator.execute(dataSource); + hashingFunction = new SubmittedVariantSummaryFunction().andThen(new SHA1HashingFunction()); clusteredHashingFunction = new ClusteredVariantSummaryFunction().andThen(new SHA1HashingFunction()); Files.deleteIfExists(this.rsReportFile.toPath()); diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/clustering_writer/MergeAccessionClusteringWriterTest.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/clustering_writer/MergeAccessionClusteringWriterTest.java index dd014267f..2d44c6621 100644 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/clustering_writer/MergeAccessionClusteringWriterTest.java +++ b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/clustering_writer/MergeAccessionClusteringWriterTest.java @@ -25,7 +25,6 @@ import org.springframework.batch.item.ItemWriter; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.data.mongodb.core.query.Query; import org.springframework.test.annotation.DirtiesContext; @@ -38,7 +37,7 @@ import uk.ac.ebi.eva.accession.clustering.batch.io.ClusteringMongoReader; import uk.ac.ebi.eva.accession.clustering.batch.io.ClusteringWriter; import uk.ac.ebi.eva.accession.clustering.test.configuration.BatchTestConfiguration; -import uk.ac.ebi.eva.accession.clustering.test.configuration.MongoTestConfiguration; +import uk.ac.ebi.eva.accession.core.configuration.ContiguousIdBlocksDataSourceConfiguration; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.ClusteredVariantAccessioningConfiguration; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.SubmittedVariantAccessioningConfiguration; import uk.ac.ebi.eva.accession.core.model.ClusteredVariant; @@ -57,6 +56,7 @@ import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantOperationEntity; import uk.ac.ebi.eva.accession.core.summary.ClusteredVariantSummaryFunction; import uk.ac.ebi.eva.accession.core.summary.SubmittedVariantSummaryFunction; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.commons.core.models.VariantType; import uk.ac.ebi.eva.commons.mongodb.readers.MongoDbCursorItemReader; @@ -94,9 +94,9 @@ * Other test classes in this folder take care of other scenarios. */ @ExtendWith(SpringExtension.class) -@EnableAutoConfiguration @ContextConfiguration(classes = {ClusteredVariantAccessioningConfiguration.class, - SubmittedVariantAccessioningConfiguration.class, BatchTestConfiguration.class, MongoTestConfiguration.class}) + SubmittedVariantAccessioningConfiguration.class, BatchTestConfiguration.class, MongoTestConfiguration.class, + ContiguousIdBlocksDataSourceConfiguration.class}) @TestPropertySource("classpath:clustering-writer-test.properties") public class MergeAccessionClusteringWriterTest extends MongoTestContainerHelper { private static final String CLUSTERED_VARIANT_COLLECTION = "clusteredVariantEntity"; diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/clustering_writer/RemappedVariantsClusteringWriterTest.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/clustering_writer/RemappedVariantsClusteringWriterTest.java index 95be884ea..02c268677 100644 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/clustering_writer/RemappedVariantsClusteringWriterTest.java +++ b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/clustering_writer/RemappedVariantsClusteringWriterTest.java @@ -26,9 +26,10 @@ import org.springframework.batch.item.ItemWriter; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.core.io.ClassPathResource; import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.jdbc.datasource.init.ResourceDatabasePopulator; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; @@ -40,7 +41,7 @@ import uk.ac.ebi.eva.accession.clustering.batch.io.RSSplitWriter; import uk.ac.ebi.eva.accession.clustering.parameters.InputParameters; import uk.ac.ebi.eva.accession.clustering.test.configuration.BatchTestConfiguration; -import uk.ac.ebi.eva.accession.clustering.test.configuration.MongoTestConfiguration; +import uk.ac.ebi.eva.accession.core.configuration.ContiguousIdBlocksDataSourceConfiguration; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.ClusteredVariantAccessioningConfiguration; import uk.ac.ebi.eva.accession.core.model.ClusteredVariant; import uk.ac.ebi.eva.accession.core.model.IClusteredVariant; @@ -53,10 +54,15 @@ import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantOperationEntity; import uk.ac.ebi.eva.accession.core.summary.ClusteredVariantSummaryFunction; import uk.ac.ebi.eva.accession.core.summary.SubmittedVariantSummaryFunction; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.commons.core.models.VariantType; import uk.ac.ebi.eva.commons.mongodb.readers.MongoDbCursorItemReader; +import javax.sql.DataSource; +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; import java.time.LocalDateTime; import java.util.ArrayList; import java.util.Arrays; @@ -84,9 +90,8 @@ * Other test classes in this folder take care of other scenarios. */ @ExtendWith(SpringExtension.class) -@EnableAutoConfiguration @ContextConfiguration(classes = {ClusteredVariantAccessioningConfiguration.class, BatchTestConfiguration.class, - MongoTestConfiguration.class}) + MongoTestConfiguration.class, ContiguousIdBlocksDataSourceConfiguration.class}) @TestPropertySource("classpath:clustering-writer-test.properties") public class RemappedVariantsClusteringWriterTest extends MongoTestContainerHelper { private static final String CLUSTERED_VARIANT_COLLECTION = "clusteredVariantEntity"; @@ -151,15 +156,27 @@ public class RemappedVariantsClusteringWriterTest extends MongoTestContainerHelp @MockBean private JobExecution jobExecution; + @Autowired + private DataSource dataSource; + private Function hashingFunction; private Function clusteredHashingFunction; @BeforeEach - public void setUp() { + public void setUp() throws SQLException { ASM_1 = inputParameters.getRemappedFrom(); ASM_2 = inputParameters.getAssemblyAccession(); mongoTemplate.getDb().drop(); + + try (Connection conn = dataSource.getConnection(); + Statement stmt = conn.createStatement()) { + stmt.execute("DROP TABLE IF EXISTS contiguous_id_blocks"); + } + ResourceDatabasePopulator populator = new ResourceDatabasePopulator(); + populator.addScript(new ClassPathResource("test-data/contiguous_id_blocks_schema.sql")); + populator.execute(dataSource); + hashingFunction = new SubmittedVariantSummaryFunction().andThen(new SHA1HashingFunction()); clusteredHashingFunction = new ClusteredVariantSummaryFunction().andThen(new SHA1HashingFunction()); diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/clustering_writer/ReuseAccessionClusteringWriterTest.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/clustering_writer/ReuseAccessionClusteringWriterTest.java index c1b132875..b792f5b34 100644 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/clustering_writer/ReuseAccessionClusteringWriterTest.java +++ b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/batch/io/clustering_writer/ReuseAccessionClusteringWriterTest.java @@ -26,11 +26,12 @@ import org.springframework.batch.item.ItemWriter; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.context.ApplicationContext; +import org.springframework.core.io.ClassPathResource; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.data.mongodb.core.query.Query; +import org.springframework.jdbc.datasource.init.ResourceDatabasePopulator; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; @@ -42,7 +43,7 @@ import uk.ac.ebi.eva.accession.clustering.metric.ClusteringMetric; import uk.ac.ebi.eva.accession.clustering.parameters.InputParameters; import uk.ac.ebi.eva.accession.clustering.test.configuration.BatchTestConfiguration; -import uk.ac.ebi.eva.accession.clustering.test.configuration.MongoTestConfiguration; +import uk.ac.ebi.eva.accession.core.configuration.ContiguousIdBlocksDataSourceConfiguration; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.ClusteredVariantAccessioningConfiguration; import uk.ac.ebi.eva.accession.core.model.ClusteredVariant; import uk.ac.ebi.eva.accession.core.model.IClusteredVariant; @@ -57,11 +58,16 @@ import uk.ac.ebi.eva.accession.core.service.nonhuman.ClusteredVariantAccessioningService; import uk.ac.ebi.eva.accession.core.summary.ClusteredVariantSummaryFunction; import uk.ac.ebi.eva.accession.core.summary.SubmittedVariantSummaryFunction; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.commons.core.models.VariantType; import uk.ac.ebi.eva.commons.mongodb.readers.MongoDbCursorItemReader; import uk.ac.ebi.eva.metrics.metric.MetricCompute; +import javax.sql.DataSource; +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -88,9 +94,8 @@ * Other test classes in this folder take care of other scenarios. */ @ExtendWith(SpringExtension.class) -@EnableAutoConfiguration @ContextConfiguration(classes = {ClusteredVariantAccessioningConfiguration.class, BatchTestConfiguration.class, - MongoTestConfiguration.class}) + MongoTestConfiguration.class, ContiguousIdBlocksDataSourceConfiguration.class}) @TestPropertySource("classpath:clustering-writer-test.properties") public class ReuseAccessionClusteringWriterTest extends MongoTestContainerHelper { private static String ASM_1; @@ -147,6 +152,10 @@ public class ReuseAccessionClusteringWriterTest extends MongoTestContainerHelper @MockBean private JobExecution jobExecution; + + @Autowired + private DataSource dataSource; + @Autowired @Qualifier(BACK_PROPAGATED_RS_WRITER) private ItemWriter backPropagatedRSWriter; @@ -156,10 +165,18 @@ public class ReuseAccessionClusteringWriterTest extends MongoTestContainerHelper private Function clusteredHashingFunction; @BeforeEach - public void setUp() { + public void setUp() throws SQLException { ASM_1 = inputParameters.getRemappedFrom(); ASM_2 = inputParameters.getAssemblyAccession(); mongoTemplate.getDb().drop(); + try (Connection conn = dataSource.getConnection(); + Statement stmt = conn.createStatement()) { + stmt.execute("DROP TABLE IF EXISTS contiguous_id_blocks"); + } + ResourceDatabasePopulator populator = new ResourceDatabasePopulator(); + populator.addScript(new ClassPathResource("test-data/contiguous_id_blocks_schema.sql")); + populator.execute(dataSource); + hashingFunction = new SubmittedVariantSummaryFunction().andThen(new SHA1HashingFunction()); clusteredHashingFunction = new ClusteredVariantSummaryFunction().andThen(new SHA1HashingFunction()); metricCompute.clearCount(); diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/ClusteringVariantJobConfigurationTest.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/ClusteringVariantJobConfigurationTest.java index f9fc8abac..016b22bb4 100644 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/ClusteringVariantJobConfigurationTest.java +++ b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/ClusteringVariantJobConfigurationTest.java @@ -25,10 +25,12 @@ import org.springframework.batch.test.JobLauncherTestUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.core.io.ClassPathResource; import org.springframework.core.io.ResourceLoader; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.http.HttpMethod; import org.springframework.http.HttpStatus; +import org.springframework.jdbc.datasource.init.ResourceDatabasePopulator; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; @@ -39,17 +41,20 @@ import uk.ac.ebi.eva.accession.clustering.configuration.batch.io.RSMergeAndSplitCandidatesReaderConfiguration; import uk.ac.ebi.eva.accession.clustering.parameters.InputParameters; import uk.ac.ebi.eva.accession.clustering.test.DatabaseState; -import uk.ac.ebi.eva.accession.clustering.test.configuration.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.accession.clustering.test.configuration.BatchTestConfiguration; -import uk.ac.ebi.eva.accession.clustering.test.configuration.MongoTestConfiguration; +import uk.ac.ebi.eva.accession.core.configuration.ContiguousIdBlocksDataSourceConfiguration; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantInactiveEntity; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantOperationEntity; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.core.utils.MongoTestDataLoader; import uk.ac.ebi.eva.metrics.count.CountServiceParameters; +import javax.sql.DataSource; import java.net.URI; +import java.sql.Connection; +import java.sql.Statement; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -78,7 +83,7 @@ @ExtendWith(SpringExtension.class) @ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class, - BatchJobRepositoryTestConfiguration.class}) + ContiguousIdBlocksDataSourceConfiguration.class}) @TestPropertySource("classpath:clustering-writer-test.properties") public class ClusteringVariantJobConfigurationTest extends MongoTestContainerHelper { @Autowired @@ -107,6 +112,9 @@ public class ClusteringVariantJobConfigurationTest extends MongoTestContainerHel @Autowired private CountServiceParameters countServiceParameters; + @Autowired + private DataSource dataSource; + @Autowired @Qualifier("COUNT_STATS_REST_TEMPLATE") private RestTemplate restTemplate; @@ -120,6 +128,14 @@ public void init() throws Exception { mockServer.expect(ExpectedCount.manyTimes(), requestTo(new URI(countServiceParameters.getUrl() + URL_PATH_SAVE_COUNT))) .andExpect(method(HttpMethod.POST)) .andRespond(withStatus(HttpStatus.OK)); + + try (Connection conn = dataSource.getConnection(); + Statement stmt = conn.createStatement()) { + stmt.execute("DROP TABLE IF EXISTS contiguous_id_blocks"); + } + ResourceDatabasePopulator populator = new ResourceDatabasePopulator(); + populator.addScript(new ClassPathResource("test-data/contiguous_id_blocks_schema.sql")); + populator.execute(dataSource); } @AfterEach diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/RSAccessionRecoveryTest.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/RSAccessionRecoveryTest.java index 7df9926c5..a976f125d 100644 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/RSAccessionRecoveryTest.java +++ b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/RSAccessionRecoveryTest.java @@ -26,22 +26,29 @@ import org.springframework.batch.test.context.SpringBatchTest; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.core.io.ClassPathResource; import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.jdbc.datasource.init.ResourceDatabasePopulator; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.ampt2d.commons.accession.persistence.jpa.monotonic.entities.ContiguousIdBlock; import uk.ac.ebi.ampt2d.commons.accession.persistence.jpa.monotonic.repositories.ContiguousIdBlockRepository; -import uk.ac.ebi.eva.accession.clustering.test.configuration.BatchJobRepositoryTestConfiguration; -import uk.ac.ebi.eva.accession.clustering.test.configuration.MongoTestConfiguration; import uk.ac.ebi.eva.accession.clustering.test.configuration.RSAccessionRecoveryTestConfiguration; +import uk.ac.ebi.eva.accession.core.configuration.ContiguousIdBlocksDataSourceConfiguration; +import uk.ac.ebi.eva.accession.core.configuration.InMemoryBatchConfiguration; import uk.ac.ebi.eva.accession.core.model.ClusteredVariant; import uk.ac.ebi.eva.accession.core.model.eva.ClusteredVariantEntity; import uk.ac.ebi.eva.accession.core.repository.nonhuman.eva.ClusteredVariantAccessioningRepository; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.commons.core.models.VariantType; +import javax.sql.DataSource; +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; import java.time.LocalDateTime; import java.util.ArrayList; import java.util.List; @@ -51,7 +58,8 @@ import static uk.ac.ebi.eva.accession.clustering.test.configuration.RSAccessionRecoveryTestConfiguration.JOB_LAUNCHER_RS_ACCESSION_RECOVERY; @ExtendWith(SpringExtension.class) -@ContextConfiguration(classes = {RSAccessionRecoveryTestConfiguration.class, MongoTestConfiguration.class, BatchJobRepositoryTestConfiguration.class}) +@ContextConfiguration(classes = {RSAccessionRecoveryTestConfiguration.class, MongoTestConfiguration.class, + ContiguousIdBlocksDataSourceConfiguration.class, InMemoryBatchConfiguration.class}) @TestPropertySource("classpath:rs-accession-recovery.properties") @SpringBatchTest public class RSAccessionRecoveryTest extends MongoTestContainerHelper { @@ -64,9 +72,20 @@ public class RSAccessionRecoveryTest extends MongoTestContainerHelper { @Autowired private MongoTemplate mongoTemplate; + @Autowired + private DataSource dataSource; + @BeforeEach - public void setUp() { + public void setUp() throws SQLException { mongoTemplate.getDb().drop(); + try (Connection conn = dataSource.getConnection(); + Statement stmt = conn.createStatement()) { + stmt.execute("DROP TABLE IF EXISTS contiguous_id_blocks"); + } + ResourceDatabasePopulator populator = new ResourceDatabasePopulator(); + populator.addScript(new ClassPathResource("test-data/contiguous_id_blocks_schema.sql")); + populator.addScript(new ClassPathResource("test-data/rs_accession_recovery_test_data.sql")); + populator.execute(dataSource); } @AfterEach @@ -203,7 +222,7 @@ private void verifyEndDBState() { assertEquals(3000000000l, block1.getFirstValue()); assertEquals(3000000029l, block1.getLastCommitted()); assertEquals(3000000029l, block1.getLastValue()); - assertEquals("0", block1.getApplicationInstanceId()); + assertEquals("1", block1.getApplicationInstanceId()); assertTrue(block1.isNotReserved()); // Block Recovered partially - (used 3000000030-3000000034 and 3000000040-3000000059) @@ -212,7 +231,7 @@ private void verifyEndDBState() { assertEquals(3000000030l, block2.getFirstValue()); assertEquals(3000000034l, block2.getLastCommitted()); assertEquals(3000000059l, block2.getLastValue()); - assertEquals("0", block2.getApplicationInstanceId()); + assertEquals("1", block2.getApplicationInstanceId()); assertTrue(block2.isNotReserved()); // Block Recovered - recovered entire block @@ -220,7 +239,7 @@ private void verifyEndDBState() { assertEquals(3000000060l, block3.getFirstValue()); assertEquals(3000000089l, block3.getLastCommitted()); assertEquals(3000000089l, block3.getLastValue()); - assertEquals("0", block3.getApplicationInstanceId()); + assertEquals("1", block3.getApplicationInstanceId()); assertTrue(block3.isNotReserved()); // Block Recovered - None of the accessions are used, just released the block @@ -228,7 +247,7 @@ private void verifyEndDBState() { assertEquals(3000000090l, block4.getFirstValue()); assertEquals(3000000089l, block4.getLastCommitted()); assertEquals(3000000119l, block4.getLastValue()); - assertEquals("0", block4.getApplicationInstanceId()); + assertEquals("1", block4.getApplicationInstanceId()); assertTrue(block4.isNotReserved()); // Block Not Recovered - Block not to be recovered as it's last update is after cut off time diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/qc/DuplicateRSAccQCStepConfigurationTest.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/qc/DuplicateRSAccQCStepConfigurationTest.java index d3b308691..fda9068d7 100644 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/qc/DuplicateRSAccQCStepConfigurationTest.java +++ b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/qc/DuplicateRSAccQCStepConfigurationTest.java @@ -30,15 +30,16 @@ import org.springframework.batch.test.JobLauncherTestUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.core.io.ClassPathResource; import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.jdbc.datasource.init.ResourceDatabasePopulator; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.ampt2d.commons.accession.hashing.SHA1HashingFunction; import uk.ac.ebi.eva.accession.clustering.batch.io.qc.DuplicateRSAccQCResult; -import uk.ac.ebi.eva.accession.clustering.test.configuration.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.accession.clustering.test.configuration.BatchTestConfiguration; -import uk.ac.ebi.eva.accession.clustering.test.configuration.MongoTestConfiguration; +import uk.ac.ebi.eva.accession.core.configuration.ContiguousIdBlocksDataSourceConfiguration; import uk.ac.ebi.eva.accession.core.model.ClusteredVariant; import uk.ac.ebi.eva.accession.core.model.IClusteredVariant; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpClusteredVariantEntity; @@ -46,12 +47,17 @@ import uk.ac.ebi.eva.accession.core.model.eva.ClusteredVariantEntity; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; import uk.ac.ebi.eva.accession.core.summary.ClusteredVariantSummaryFunction; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.commons.core.models.pipeline.Variant; +import javax.sql.DataSource; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Paths; +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; import java.util.Arrays; import java.util.HashSet; import java.util.List; @@ -66,7 +72,8 @@ import static uk.ac.ebi.eva.accession.clustering.test.configuration.BatchTestConfiguration.JOB_LAUNCHER_DUPLICATE_RS_ACC_QC_JOB; @ExtendWith(SpringExtension.class) -@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class, BatchJobRepositoryTestConfiguration.class}) +@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class, + ContiguousIdBlocksDataSourceConfiguration.class}) @TestPropertySource("classpath:duplicate-rs-acc-qc-test.properties") public class DuplicateRSAccQCStepConfigurationTest extends MongoTestContainerHelper { private static final String duplicateRsAccFile = "src/test/resources/duplicateRSAcc.csv"; @@ -78,12 +85,23 @@ public class DuplicateRSAccQCStepConfigurationTest extends MongoTestContainerHel @Autowired private MongoTemplate mongoTemplate; + @Autowired + private DataSource dataSource; + private final Gson gson = new GsonBuilder().create(); @BeforeEach - public void setUp() throws IOException { + public void setUp() throws IOException, SQLException { mongoTemplate.getDb().drop(); Files.deleteIfExists(Paths.get(duplicateRsAccFile)); + + try (Connection conn = dataSource.getConnection(); + Statement stmt = conn.createStatement()) { + stmt.execute("DROP TABLE IF EXISTS contiguous_id_blocks"); + } + ResourceDatabasePopulator populator = new ResourceDatabasePopulator(); + populator.addScript(new ClassPathResource("test-data/contiguous_id_blocks_schema.sql")); + populator.execute(dataSource); } @AfterEach diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/qc/NewClusteredVariantsQCJobConfigurationTest.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/qc/NewClusteredVariantsQCJobConfigurationTest.java index 91b020eb3..c5cdbd8a7 100644 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/qc/NewClusteredVariantsQCJobConfigurationTest.java +++ b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/qc/NewClusteredVariantsQCJobConfigurationTest.java @@ -30,9 +30,9 @@ import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit.jupiter.SpringExtension; -import uk.ac.ebi.eva.accession.clustering.test.configuration.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.accession.clustering.test.configuration.BatchTestConfiguration; -import uk.ac.ebi.eva.accession.clustering.test.configuration.MongoTestConfiguration; +import uk.ac.ebi.eva.accession.core.configuration.ContiguousIdBlocksDataSourceConfiguration; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.core.utils.MongoTestDataLoader; @@ -46,7 +46,8 @@ import static uk.ac.ebi.eva.accession.clustering.test.configuration.BatchTestConfiguration.JOB_LAUNCHER_NEW_CLUSTERED_VARIANTS_QC; @ExtendWith(SpringExtension.class) -@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class, BatchJobRepositoryTestConfiguration.class}) +@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class, + ContiguousIdBlocksDataSourceConfiguration.class}) @TestPropertySource("classpath:clustering-qc-test.properties") public class NewClusteredVariantsQCJobConfigurationTest extends MongoTestContainerHelper { diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/steps/ClusteringVariantStepConfigurationTest.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/steps/ClusteringVariantStepConfigurationTest.java index 2aa9ddbe5..4fcf32e3f 100644 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/steps/ClusteringVariantStepConfigurationTest.java +++ b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/steps/ClusteringVariantStepConfigurationTest.java @@ -28,10 +28,12 @@ import org.springframework.batch.test.JobLauncherTestUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.core.io.ClassPathResource; import org.springframework.core.io.ResourceLoader; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.http.HttpMethod; import org.springframework.http.HttpStatus; +import org.springframework.jdbc.datasource.init.ResourceDatabasePopulator; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; @@ -40,15 +42,18 @@ import org.springframework.test.web.client.MockRestServiceServer; import org.springframework.web.client.RestTemplate; import uk.ac.ebi.ampt2d.commons.accession.core.models.EventType; -import uk.ac.ebi.eva.accession.clustering.test.configuration.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.accession.clustering.test.configuration.BatchTestConfiguration; -import uk.ac.ebi.eva.accession.clustering.test.configuration.MongoTestConfiguration; +import uk.ac.ebi.eva.accession.core.configuration.ContiguousIdBlocksDataSourceConfiguration; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantOperationEntity; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.core.utils.MongoTestDataLoader; import uk.ac.ebi.eva.metrics.count.CountServiceParameters; +import javax.sql.DataSource; import java.net.URI; +import java.sql.Connection; +import java.sql.Statement; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -68,7 +73,7 @@ @ExtendWith(SpringExtension.class) @ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class, - BatchJobRepositoryTestConfiguration.class}) + ContiguousIdBlocksDataSourceConfiguration.class}) @TestPropertySource("classpath:clustering-issuance-test.properties") public class ClusteringVariantStepConfigurationTest extends MongoTestContainerHelper { private static final String CLUSTERED_VARIANT_COLLECTION = "clusteredVariantEntity"; @@ -90,6 +95,9 @@ public class ClusteringVariantStepConfigurationTest extends MongoTestContainerHe @Autowired private MongoTemplate mongoTemplate; + @Autowired + private DataSource dataSource; + @Autowired private ResourceLoader resourceLoader; @@ -112,6 +120,14 @@ public void init() throws Exception { mockServer.expect(ExpectedCount.manyTimes(), requestTo(new URI(countServiceParameters.getUrl() + URL_PATH_SAVE_COUNT))) .andExpect(method(HttpMethod.POST)) .andRespond(withStatus(HttpStatus.OK)); + + try (Connection conn = dataSource.getConnection(); + Statement stmt = conn.createStatement()) { + stmt.execute("DROP TABLE IF EXISTS contiguous_id_blocks"); + } + ResourceDatabasePopulator populator = new ResourceDatabasePopulator(); + populator.addScript(new ClassPathResource("test-data/contiguous_id_blocks_schema.sql")); + populator.execute(dataSource); } @AfterEach diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/runner/ClusteringCommandLineRunnerTest.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/runner/ClusteringCommandLineRunnerTest.java index 7206b6cf1..97cc522a2 100644 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/runner/ClusteringCommandLineRunnerTest.java +++ b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/runner/ClusteringCommandLineRunnerTest.java @@ -25,7 +25,6 @@ import org.mockito.Mockito; import org.springframework.batch.core.JobExecution; import org.springframework.batch.core.JobExecutionException; -import org.springframework.batch.core.JobInstance; import org.springframework.batch.core.explore.JobExplorer; import org.springframework.batch.core.repository.JobRepository; import org.springframework.batch.item.Chunk; @@ -36,11 +35,13 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.core.io.ClassPathResource; import org.springframework.core.io.ResourceLoader; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.data.mongodb.core.query.Query; import org.springframework.http.HttpMethod; import org.springframework.http.HttpStatus; +import org.springframework.jdbc.datasource.init.ResourceDatabasePopulator; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; @@ -62,9 +63,8 @@ import uk.ac.ebi.eva.accession.clustering.configuration.batch.io.RSMergeAndSplitCandidatesReaderConfiguration; import uk.ac.ebi.eva.accession.clustering.parameters.InputParameters; import uk.ac.ebi.eva.accession.clustering.test.DatabaseState; -import uk.ac.ebi.eva.accession.clustering.test.configuration.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.accession.clustering.test.configuration.BatchTestConfiguration; -import uk.ac.ebi.eva.accession.clustering.test.configuration.MongoTestConfiguration; +import uk.ac.ebi.eva.accession.core.configuration.ContiguousIdBlocksDataSourceConfiguration; import uk.ac.ebi.eva.accession.core.model.ClusteredVariant; import uk.ac.ebi.eva.accession.core.model.IClusteredVariant; import uk.ac.ebi.eva.accession.core.model.ISubmittedVariant; @@ -77,11 +77,11 @@ import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantInactiveEntity; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantOperationEntity; -import uk.ac.ebi.eva.accession.core.runner.CommandLineRunnerUtils; import uk.ac.ebi.eva.accession.core.service.nonhuman.ClusteredVariantAccessioningService; import uk.ac.ebi.eva.accession.core.service.nonhuman.SubmittedVariantAccessioningService; import uk.ac.ebi.eva.accession.core.summary.ClusteredVariantSummaryFunction; import uk.ac.ebi.eva.accession.core.summary.SubmittedVariantSummaryFunction; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.core.utils.MongoTestDataLoader; import uk.ac.ebi.eva.commons.core.models.VariantClassifier; @@ -89,8 +89,11 @@ import uk.ac.ebi.eva.commons.mongodb.readers.MongoDbCursorItemReader; import uk.ac.ebi.eva.metrics.count.CountServiceParameters; +import javax.sql.DataSource; import java.io.File; import java.net.URI; +import java.sql.Connection; +import java.sql.Statement; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -108,24 +111,17 @@ import static org.springframework.test.web.client.match.MockRestRequestMatchers.method; import static org.springframework.test.web.client.match.MockRestRequestMatchers.requestTo; import static org.springframework.test.web.client.response.MockRestResponseCreators.withStatus; -import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.BACK_PROPAGATE_NEW_RS_STEP; -import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.BACK_PROPAGATE_SPLIT_OR_MERGED_RS_STEP; -import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.CLEAR_RS_MERGE_AND_SPLIT_CANDIDATES_STEP; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.CLUSTERED_CLUSTERING_WRITER; -import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.CLUSTERING_CLUSTERED_VARIANTS_FROM_MONGO_STEP; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.CLUSTERING_FROM_MONGO_JOB; -import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.CLUSTERING_NON_CLUSTERED_VARIANTS_FROM_MONGO_STEP; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.CLUSTER_UNCLUSTERED_VARIANTS_JOB; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.PROCESS_REMAPPED_VARIANTS_WITH_RS_JOB; -import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.PROCESS_RS_MERGE_CANDIDATES_STEP; -import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.PROCESS_RS_SPLIT_CANDIDATES_STEP; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.RS_MERGE_WRITER; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.RS_SPLIT_WRITER; import static uk.ac.ebi.eva.accession.clustering.test.configuration.BatchTestConfiguration.JOB_LAUNCHER_FROM_MONGO_ONLY_FIRST_STEP; @ExtendWith(SpringExtension.class) @ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class, - BatchJobRepositoryTestConfiguration.class}) + ContiguousIdBlocksDataSourceConfiguration.class}) @TestPropertySource("classpath:clustering-pipeline-test.properties") public class ClusteringCommandLineRunnerTest extends MongoTestContainerHelper { private static String TEST_APPLICATION_INSTANCE_ID = "test-application-instance-id"; @@ -203,6 +199,9 @@ public class ClusteringCommandLineRunnerTest extends MongoTestContainerHelper { @Qualifier(RS_SPLIT_WRITER) private RSSplitWriter rsSplitWriter; + @Autowired + private DataSource dataSource; + @MockBean private JobExecution jobExecution; @@ -274,7 +273,6 @@ public void setUp() throws Exception { ASM2 = inputParameters.getAssemblyAccession(); jobRepositoryTestUtils = new JobRepositoryTestUtils(jobRepository); jobRepositoryTestUtils.removeJobExecutions(); - inputParameters.setForceRestart(false); inputParameters.setRemappedFrom(originalRemappedFrom); mockServer = MockRestServiceServer.createServer(restTemplate); @@ -282,6 +280,14 @@ public void setUp() throws Exception { .andExpect(method(HttpMethod.POST)) .andRespond(withStatus(HttpStatus.OK)); mongoTemplate.getDb().drop(); + try (Connection conn = dataSource.getConnection(); + Statement stmt = conn.createStatement()) { + stmt.execute("DROP TABLE IF EXISTS contiguous_id_blocks"); + } + ResourceDatabasePopulator populator = new ResourceDatabasePopulator(); + populator.addScript(new ClassPathResource("test-data/contiguous_id_blocks_schema.sql")); + populator.execute(dataSource); + Mockito.when(jobExecution.getJobId()).thenReturn(1L); rsSplitWriter.setJobExecution(jobExecution); @@ -290,7 +296,6 @@ public void setUp() throws Exception { @AfterEach public void tearDown() { jobRepositoryTestUtils.removeJobExecutions(); - inputParameters.setForceRestart(false); mongoTemplate.getDb().drop(); } @@ -302,25 +307,6 @@ public void runFullClusteringFromMongoJobWithNoErrors() throws JobExecutionExcep runner.setJobName(CLUSTERING_FROM_MONGO_JOB); runner.run(); assertEquals(ClusteringCommandLineRunner.EXIT_WITHOUT_ERRORS, runner.getExitCode()); - - JobInstance currentJobInstance = - CommandLineRunnerUtils.getLastJobExecution(CLUSTERING_FROM_MONGO_JOB, - jobExplorer, inputParameters.toJobParameters()) - .getJobInstance(); - assertEquals(1, jobRepository.getStepExecutionCount(currentJobInstance, - CLUSTERING_CLUSTERED_VARIANTS_FROM_MONGO_STEP)); - assertEquals(1, jobRepository.getStepExecutionCount(currentJobInstance, - PROCESS_RS_MERGE_CANDIDATES_STEP)); - assertEquals(1, jobRepository.getStepExecutionCount(currentJobInstance, - PROCESS_RS_SPLIT_CANDIDATES_STEP)); - assertEquals(1, jobRepository.getStepExecutionCount(currentJobInstance, - CLEAR_RS_MERGE_AND_SPLIT_CANDIDATES_STEP)); - assertEquals(1, jobRepository.getStepExecutionCount(currentJobInstance, - CLUSTERING_NON_CLUSTERED_VARIANTS_FROM_MONGO_STEP)); - assertEquals(1, jobRepository.getStepExecutionCount(currentJobInstance, - BACK_PROPAGATE_NEW_RS_STEP)); - assertEquals(1, jobRepository.getStepExecutionCount(currentJobInstance, - BACK_PROPAGATE_SPLIT_OR_MERGED_RS_STEP)); } @Test @@ -334,24 +320,6 @@ public void runPartialClusteringFromMongoJobForNonRemappedVariants() throws JobE runner.setJobName(CLUSTERING_FROM_MONGO_JOB); runner.run(); assertEquals(ClusteringCommandLineRunner.EXIT_WITHOUT_ERRORS, runner.getExitCode()); - - JobInstance currentJobInstance = - CommandLineRunnerUtils.getLastJobExecution(CLUSTERING_FROM_MONGO_JOB, - jobExplorer, inputParameters.toJobParameters()).getJobInstance(); - assertEquals(0, jobRepository.getStepExecutionCount(currentJobInstance, - CLUSTERING_CLUSTERED_VARIANTS_FROM_MONGO_STEP)); - assertEquals(0, jobRepository.getStepExecutionCount(currentJobInstance, - PROCESS_RS_MERGE_CANDIDATES_STEP)); - assertEquals(0, jobRepository.getStepExecutionCount(currentJobInstance, - PROCESS_RS_SPLIT_CANDIDATES_STEP)); - assertEquals(0, jobRepository.getStepExecutionCount(currentJobInstance, - CLEAR_RS_MERGE_AND_SPLIT_CANDIDATES_STEP)); - assertEquals(1, jobRepository.getStepExecutionCount(currentJobInstance, - CLUSTERING_NON_CLUSTERED_VARIANTS_FROM_MONGO_STEP)); - assertEquals(0, jobRepository.getStepExecutionCount(currentJobInstance, - BACK_PROPAGATE_NEW_RS_STEP)); - assertEquals(0, jobRepository.getStepExecutionCount(currentJobInstance, - BACK_PROPAGATE_SPLIT_OR_MERGED_RS_STEP)); } @Test @@ -920,30 +888,6 @@ public void runNonExistentJob() throws JobExecutionException { assertEquals(ClusteringCommandLineRunner.EXIT_WITH_ERRORS, runner.getExitCode()); } - @Test - @DirtiesContext - public void restartCompletedJobThatIsAlreadyInTheRepository() throws Exception { - runner.setJobName(CLUSTERING_FROM_MONGO_JOB); - runner.run(); - assertEquals(ClusteringCommandLineRunner.EXIT_WITHOUT_ERRORS, runner.getExitCode()); - - inputParameters.setForceRestart(true); - runner.run(); - assertEquals(ClusteringCommandLineRunner.EXIT_WITHOUT_ERRORS, runner.getExitCode()); - } - - - @Test - @DirtiesContext - public void forceRestartButNoJobInTheRepository() throws Exception { - runner.setJobName(CLUSTERING_FROM_MONGO_JOB); - inputParameters.setForceRestart(true); - assertEquals(Collections.EMPTY_LIST, jobExplorer.getJobNames()); - runner.run(); - - assertEquals(ClusteringCommandLineRunner.EXIT_WITHOUT_ERRORS, runner.getExitCode()); - } - @Test @DirtiesContext // See https://docs.google.com/spreadsheets/d/1KQLVCUy-vqXKgkCDt2czX6kuMfsjfCc9uBsS19MZ6dY/edit#rangeid=972582890 diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/test/configuration/BatchJobRepositoryTestConfiguration.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/test/configuration/BatchJobRepositoryTestConfiguration.java deleted file mode 100644 index 6cebe2481..000000000 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/test/configuration/BatchJobRepositoryTestConfiguration.java +++ /dev/null @@ -1,22 +0,0 @@ -package uk.ac.ebi.eva.accession.clustering.test.configuration; - -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Primary; -import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseBuilder; -import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseType; - -import javax.sql.DataSource; - -@Configuration -public class BatchJobRepositoryTestConfiguration { - - @Bean - @Primary - public DataSource dataSource() { - return new EmbeddedDatabaseBuilder() - .setType(EmbeddedDatabaseType.HSQL) - .addScript("org/springframework/batch/core/schema-hsqldb.sql") - .build(); - } -} \ No newline at end of file diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/test/configuration/BatchTestConfiguration.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/test/configuration/BatchTestConfiguration.java index cc5eb69f8..40f129633 100644 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/test/configuration/BatchTestConfiguration.java +++ b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/test/configuration/BatchTestConfiguration.java @@ -23,10 +23,10 @@ import org.springframework.batch.test.JobLauncherTestUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Import; import org.springframework.core.io.ResourceLoader; +import org.springframework.web.client.RestTemplate; import uk.ac.ebi.eva.accession.clustering.configuration.batch.io.BackPropagatedRSWriterConfiguration; import uk.ac.ebi.eva.accession.clustering.configuration.batch.io.ClusteringMongoReaderConfiguration; import uk.ac.ebi.eva.accession.clustering.configuration.batch.io.ClusteringWriterConfiguration; @@ -55,6 +55,7 @@ import uk.ac.ebi.eva.accession.clustering.configuration.batch.steps.RSAccessionRecoveryStepConfiguration; import uk.ac.ebi.eva.accession.clustering.configuration.batch.steps.qc.DuplicateRSAccQCStepConfiguration; import uk.ac.ebi.eva.accession.clustering.runner.ClusteringCommandLineRunner; +import uk.ac.ebi.eva.accession.core.configuration.InMemoryBatchConfiguration; import uk.ac.ebi.eva.commons.batch.job.JobExecutionApplicationListener; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.CLUSTERING_FROM_MONGO_JOB; @@ -64,8 +65,8 @@ import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.STUDY_CLUSTERING_JOB; import static uk.ac.ebi.eva.accession.clustering.configuration.batch.jobs.qc.NewClusteredVariantsQCJobConfiguration.NEW_CLUSTERED_VARIANTS_QC_JOB; -@EnableAutoConfiguration -@Import({ClusteringFromMongoJobConfiguration.class, +@Import({InMemoryBatchConfiguration.class, + ClusteringFromMongoJobConfiguration.class, StudyClusteringJobConfiguration.class, NewClusteredVariantsQCJobConfiguration.class, ProcessRemappedVariantsWithRSJobConfiguration.class, @@ -182,4 +183,10 @@ public JobLauncherTestUtils jobLauncherTestUtilsDuplicateRSAccQCJob(JobLauncher utils.setJob(job); return utils; } + + @Bean(name = "COUNT_STATS_REST_TEMPLATE") + public RestTemplate countStatsRestTemplate() { + return new RestTemplate(); + } + } diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/test/configuration/MongoTestConfiguration.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/test/configuration/MongoTestConfiguration.java deleted file mode 100644 index a4a8fc2c0..000000000 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/test/configuration/MongoTestConfiguration.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Copyright 2020 EMBL - European Bioinformatics Institute - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package uk.ac.ebi.eva.accession.clustering.test.configuration; - -import com.mongodb.ReadPreference; -import com.mongodb.WriteConcern; -import com.mongodb.client.MongoClient; -import com.mongodb.client.MongoClients; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.domain.EntityScan; -import org.springframework.boot.test.autoconfigure.data.mongo.AutoConfigureDataMongo; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Primary; -import org.springframework.data.mongodb.MongoDatabaseFactory; -import org.springframework.data.mongodb.config.EnableMongoAuditing; -import org.springframework.data.mongodb.core.MongoTemplate; -import org.springframework.data.mongodb.core.SimpleMongoClientDatabaseFactory; -import org.springframework.data.mongodb.core.WriteResultChecking; -import org.springframework.data.mongodb.core.convert.DefaultDbRefResolver; -import org.springframework.data.mongodb.core.convert.DefaultMongoTypeMapper; -import org.springframework.data.mongodb.core.convert.MappingMongoConverter; -import org.springframework.data.mongodb.core.convert.MongoCustomConversions; -import org.springframework.data.mongodb.core.mapping.MongoMappingContext; -import org.springframework.data.mongodb.repository.config.EnableMongoRepositories; - -import java.util.Collections; - -@Configuration -@EntityScan(basePackages = {"uk.ac.ebi.eva.accession.core.persistence"}) -@EnableMongoRepositories(basePackages = { - "uk.ac.ebi.eva.accession.core.persistence", - "uk.ac.ebi.ampt2d.commons.accession.persistence.mongodb.repository"}) -@EnableMongoAuditing -@AutoConfigureDataMongo -public class MongoTestConfiguration { - - @Value("${spring.data.mongodb.uri}") - private String mongoUri; - - @Value("${spring.data.mongodb.database}") - private String database; - - @Value("${mongodb.read-preference:primary}") - private String readPreference; - - @Bean - @Primary - public MongoClient mongoClient() { - return MongoClients.create(mongoUri); - } - - @Bean("primaryFactory") - @Primary - public MongoDatabaseFactory mongoDbFactory() { - return new SimpleMongoClientDatabaseFactory(mongoClient(), database); - } - - @Bean - @Primary - public MongoCustomConversions mongoCustomConversions() { - return new MongoCustomConversions(Collections.emptyList()); - } - - @Bean - @Primary - public MongoMappingContext mongoMappingContext(MongoCustomConversions conversions) { - MongoMappingContext context = new MongoMappingContext(); - context.setSimpleTypeHolder(conversions.getSimpleTypeHolder()); - context.setAutoIndexCreation(false); - return context; - } - - @Bean - @Primary - public MongoTemplate mongoTemplate(MongoMappingContext mongoMappingContext, - MongoCustomConversions conversions) { - MappingMongoConverter converter = new MappingMongoConverter( - new DefaultDbRefResolver(mongoDbFactory()), mongoMappingContext); - converter.setCustomConversions(conversions); - converter.setTypeMapper(new DefaultMongoTypeMapper(null)); - converter.afterPropertiesSet(); - MongoTemplate mongoTemplate = new MongoTemplate(mongoDbFactory(), converter); - mongoTemplate.setReadPreference(ReadPreference.valueOf(readPreference)); - mongoTemplate.setWriteConcern(WriteConcern.MAJORITY); - mongoTemplate.setWriteResultChecking(WriteResultChecking.EXCEPTION); - return mongoTemplate; - } -} \ No newline at end of file diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/test/configuration/RSAccessionRecoveryTestConfiguration.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/test/configuration/RSAccessionRecoveryTestConfiguration.java index 6cd5d315c..0c73614a9 100644 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/test/configuration/RSAccessionRecoveryTestConfiguration.java +++ b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/test/configuration/RSAccessionRecoveryTestConfiguration.java @@ -21,7 +21,6 @@ import org.springframework.batch.core.repository.JobRepository; import org.springframework.batch.test.JobLauncherTestUtils; import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Import; import uk.ac.ebi.eva.accession.clustering.configuration.batch.jobs.RSAccessionRecoveryJobConfiguration; @@ -32,7 +31,6 @@ import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.RS_ACCESSION_RECOVERY_JOB; -@EnableAutoConfiguration @Import({RSAccessionRecoveryJobConfiguration.class, RSAccessionRecoveryStepConfiguration.class, RSAccessionRecoveryServiceConfiguration.class, diff --git a/eva-accession-clustering/src/test/resources/backpropagation-test.properties b/eva-accession-clustering/src/test/resources/backpropagation-test.properties index 336a50626..fcdc8ba42 100644 --- a/eva-accession-clustering/src/test/resources/backpropagation-test.properties +++ b/eva-accession-clustering/src/test/resources/backpropagation-test.properties @@ -21,6 +21,10 @@ accessioning.monotonic.rs.blockSize=100000 accessioning.monotonic.rs.blockStartValue=5 accessioning.monotonic.rs.nextBlockInterval=1000000000 +continuous.id.blocks.datasource.driver-class-name=org.hsqldb.jdbc.JDBCDriver +continuous.id.blocks.datasource.jdbc-url=jdbc:hsqldb:mem:db;sql.syntax_pgs=true;DB_CLOSE_DELAY=-1 +continuous.id.blocks.datasource.username=SA +continuous.id.blocks.datasource.password= spring.data.mongodb.database=test-db mongodb.read-preference=primary diff --git a/eva-accession-clustering/src/test/resources/clustering-issuance-test.properties b/eva-accession-clustering/src/test/resources/clustering-issuance-test.properties index d39fc667d..07a0d74bf 100644 --- a/eva-accession-clustering/src/test/resources/clustering-issuance-test.properties +++ b/eva-accession-clustering/src/test/resources/clustering-issuance-test.properties @@ -22,6 +22,11 @@ accessioning.monotonic.rs.blockSize=100000 accessioning.monotonic.rs.blockStartValue=3000000000 accessioning.monotonic.rs.nextBlockInterval=1000000000 +continuous.id.blocks.datasource.driver-class-name=org.hsqldb.jdbc.JDBCDriver +continuous.id.blocks.datasource.jdbc-url=jdbc:hsqldb:mem:db;sql.syntax_pgs=true;DB_CLOSE_DELAY=-1 +continuous.id.blocks.datasource.username=SA +continuous.id.blocks.datasource.password= + spring.data.mongodb.database=test-db mongodb.read-preference=primary diff --git a/eva-accession-clustering/src/test/resources/clustering-pipeline-test.properties b/eva-accession-clustering/src/test/resources/clustering-pipeline-test.properties index cadc78ebb..5b3cd35fd 100644 --- a/eva-accession-clustering/src/test/resources/clustering-pipeline-test.properties +++ b/eva-accession-clustering/src/test/resources/clustering-pipeline-test.properties @@ -22,6 +22,11 @@ accessioning.monotonic.rs.blockSize=100000 accessioning.monotonic.rs.blockStartValue=5 accessioning.monotonic.rs.nextBlockInterval=1000000000 +continuous.id.blocks.datasource.driver-class-name=org.hsqldb.jdbc.JDBCDriver +continuous.id.blocks.datasource.jdbc-url=jdbc:hsqldb:mem:db;sql.syntax_pgs=true;DB_CLOSE_DELAY=-1 +continuous.id.blocks.datasource.username=SA +continuous.id.blocks.datasource.password= + spring.data.mongodb.database=test-db mongodb.read-preference=primary diff --git a/eva-accession-clustering/src/test/resources/clustering-qc-test.properties b/eva-accession-clustering/src/test/resources/clustering-qc-test.properties index 4b654722b..50c6eb203 100644 --- a/eva-accession-clustering/src/test/resources/clustering-qc-test.properties +++ b/eva-accession-clustering/src/test/resources/clustering-qc-test.properties @@ -21,6 +21,11 @@ accessioning.monotonic.rs.blockSize=100000 accessioning.monotonic.rs.blockStartValue=3000000000 accessioning.monotonic.rs.nextBlockInterval=1000000000 +continuous.id.blocks.datasource.driver-class-name=org.hsqldb.jdbc.JDBCDriver +continuous.id.blocks.datasource.jdbc-url=jdbc:hsqldb:mem:db;sql.syntax_pgs=true;DB_CLOSE_DELAY=-1 +continuous.id.blocks.datasource.username=SA +continuous.id.blocks.datasource.password= + spring.data.mongodb.database=test-db mongodb.read-preference=primary diff --git a/eva-accession-clustering/src/test/resources/clustering-writer-test.properties b/eva-accession-clustering/src/test/resources/clustering-writer-test.properties index 652d34dd3..1e68809a3 100644 --- a/eva-accession-clustering/src/test/resources/clustering-writer-test.properties +++ b/eva-accession-clustering/src/test/resources/clustering-writer-test.properties @@ -22,6 +22,11 @@ accessioning.monotonic.rs.blockSize=100000 accessioning.monotonic.rs.blockStartValue=3000000000 accessioning.monotonic.rs.nextBlockInterval=1000000000 +continuous.id.blocks.datasource.driver-class-name=org.hsqldb.jdbc.JDBCDriver +continuous.id.blocks.datasource.jdbc-url=jdbc:hsqldb:mem:db;sql.syntax_pgs=true;DB_CLOSE_DELAY=-1 +continuous.id.blocks.datasource.username=SA +continuous.id.blocks.datasource.password= + spring.data.mongodb.database=test-db mongodb.read-preference=primary diff --git a/eva-accession-clustering/src/test/resources/duplicate-rs-acc-qc-test.properties b/eva-accession-clustering/src/test/resources/duplicate-rs-acc-qc-test.properties index 6c0cf5110..7fcdc30ea 100644 --- a/eva-accession-clustering/src/test/resources/duplicate-rs-acc-qc-test.properties +++ b/eva-accession-clustering/src/test/resources/duplicate-rs-acc-qc-test.properties @@ -23,6 +23,11 @@ accessioning.monotonic.rs.blockSize=100000 accessioning.monotonic.rs.blockStartValue=5 accessioning.monotonic.rs.nextBlockInterval=1000000000 +continuous.id.blocks.datasource.driver-class-name=org.hsqldb.jdbc.JDBCDriver +continuous.id.blocks.datasource.jdbc-url=jdbc:hsqldb:mem:db;sql.syntax_pgs=true;DB_CLOSE_DELAY=-1 +continuous.id.blocks.datasource.username=SA +continuous.id.blocks.datasource.password= + spring.data.mongodb.database=test-db mongodb.read-preference=primary diff --git a/eva-accession-clustering/src/test/resources/merge-split-test.properties b/eva-accession-clustering/src/test/resources/merge-split-test.properties index 3aea699b4..450da55cf 100644 --- a/eva-accession-clustering/src/test/resources/merge-split-test.properties +++ b/eva-accession-clustering/src/test/resources/merge-split-test.properties @@ -20,6 +20,11 @@ accessioning.monotonic.rs.blockSize=100000 accessioning.monotonic.rs.blockStartValue=4 accessioning.monotonic.rs.nextBlockInterval=1000000000 +continuous.id.blocks.datasource.driver-class-name=org.hsqldb.jdbc.JDBCDriver +continuous.id.blocks.datasource.jdbc-url=jdbc:hsqldb:mem:db;sql.syntax_pgs=true;DB_CLOSE_DELAY=-1 +continuous.id.blocks.datasource.username=SA +continuous.id.blocks.datasource.password= + spring.data.mongodb.database=test-db mongodb.read-preference=primary diff --git a/eva-accession-clustering/src/test/resources/rs-accession-recovery.properties b/eva-accession-clustering/src/test/resources/rs-accession-recovery.properties index b5418f3f3..0bd65e138 100644 --- a/eva-accession-clustering/src/test/resources/rs-accession-recovery.properties +++ b/eva-accession-clustering/src/test/resources/rs-accession-recovery.properties @@ -1,13 +1,9 @@ recovery.cutoff.days=14 -spring.datasource.driver-class-name=org.hsqldb.jdbcDriver -spring.datasource.url=jdbc:hsqldb:mem:db;sql.syntax_pgs=true;DB_CLOSE_DELAY=-1 -spring.datasource.username=SA -spring.datasource.password= -spring.sql.init.schema-locations=classpath:test-data/contiguous_id_blocks_schema.sql -spring.sql.init.data-locations=classpath:test-data/rs_accession_recovery_test_data.sql -spring.sql.init.mode=always -spring.jpa.hibernate.ddl-auto=none +continuous.id.blocks.datasource.driver-class-name=org.hsqldb.jdbc.JDBCDriver +continuous.id.blocks.datasource.jdbc-url=jdbc:hsqldb:mem:db;sql.syntax_pgs=true;DB_CLOSE_DELAY=-1 +continuous.id.blocks.datasource.username=SA +continuous.id.blocks.datasource.password= parameters.vcf=src/test/resources/input-files/vcf/aggregated_accessioned.vcf.gz parameters.projectAccession=projectId_1 diff --git a/eva-accession-clustering/src/test/resources/test-data/contiguous_id_blocks_schema.sql b/eva-accession-clustering/src/test/resources/test-data/contiguous_id_blocks_schema.sql index 495f9a4fb..a89a52a6b 100644 --- a/eva-accession-clustering/src/test/resources/test-data/contiguous_id_blocks_schema.sql +++ b/eva-accession-clustering/src/test/resources/test-data/contiguous_id_blocks_schema.sql @@ -1,3 +1,7 @@ +CREATE SEQUENCE IF NOT EXISTS contiguous_id_blocks_seq + START WITH 1 + INCREMENT BY 50; + CREATE TABLE contiguous_id_blocks ( id bigint not NULL, application_instance_id varchar(255) not NULL, diff --git a/eva-accession-core/pom.xml b/eva-accession-core/pom.xml index 9335ac7ea..ffa56fdfe 100644 --- a/eva-accession-core/pom.xml +++ b/eva-accession-core/pom.xml @@ -36,10 +36,6 @@ org.springframework.boot spring-boot-starter-data-mongodb - - org.postgresql - postgresql - uk.ac.ebi.ampt2d accession-commons-mongodb @@ -48,6 +44,11 @@ uk.ac.ebi.ampt2d accession-commons-monotonic-generator-jpa + + org.hsqldb + hsqldb + test + org.springframework.boot spring-boot-testcontainers diff --git a/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/GenericApplication.java b/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/GenericApplication.java index 4053156ff..199dc448a 100644 --- a/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/GenericApplication.java +++ b/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/GenericApplication.java @@ -15,13 +15,22 @@ */ package uk.ac.ebi.eva.accession.core; -import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.autoconfigure.batch.BatchAutoConfiguration; +import org.springframework.boot.autoconfigure.data.mongo.MongoDataAutoConfiguration; +import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration; +import org.springframework.boot.autoconfigure.jdbc.DataSourceTransactionManagerAutoConfiguration; +import org.springframework.boot.autoconfigure.orm.jpa.HibernateJpaAutoConfiguration; import org.springframework.context.ConfigurableApplicationContext; -@EnableBatchProcessing -@SpringBootApplication +@SpringBootApplication(exclude = { + MongoDataAutoConfiguration.class, + DataSourceAutoConfiguration.class, + DataSourceTransactionManagerAutoConfiguration.class, + HibernateJpaAutoConfiguration.class, + BatchAutoConfiguration.class +}) public class GenericApplication { public static void main(String[] args) { diff --git a/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/configuration/ContiguousIdBlocksDataSourceConfiguration.java b/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/configuration/ContiguousIdBlocksDataSourceConfiguration.java new file mode 100644 index 000000000..1cbe609f2 --- /dev/null +++ b/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/configuration/ContiguousIdBlocksDataSourceConfiguration.java @@ -0,0 +1,50 @@ +package uk.ac.ebi.eva.accession.core.configuration; + +import jakarta.persistence.EntityManagerFactory; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.boot.jdbc.DataSourceBuilder; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Primary; +import org.springframework.orm.jpa.JpaTransactionManager; +import org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean; +import org.springframework.orm.jpa.vendor.HibernateJpaVendorAdapter; +import org.springframework.transaction.PlatformTransactionManager; + +import javax.sql.DataSource; +import java.util.Properties; + +@Configuration +public class ContiguousIdBlocksDataSourceConfiguration { + + @Bean + @Primary + @ConfigurationProperties(prefix = "continuous.id.blocks.datasource") + public DataSource continuousIdBlocksDataSource() { + return DataSourceBuilder.create().build(); + } + + @Bean + @Primary + public LocalContainerEntityManagerFactoryBean entityManagerFactory(@Qualifier("continuousIdBlocksDataSource") + DataSource dataSource) { + LocalContainerEntityManagerFactoryBean em = new LocalContainerEntityManagerFactoryBean(); + em.setDataSource(dataSource); + em.setPackagesToScan("uk.ac.ebi.ampt2d.commons.accession.persistence.jpa.monotonic.entities"); + HibernateJpaVendorAdapter vendorAdapter = new HibernateJpaVendorAdapter(); + em.setJpaVendorAdapter(vendorAdapter); + + Properties jpaProperties = new Properties(); + jpaProperties.setProperty("hibernate.physical_naming_strategy", + "org.hibernate.boot.model.naming.CamelCaseToUnderscoresNamingStrategy"); + em.setJpaProperties(jpaProperties); + return em; + } + + @Bean + @Primary + public PlatformTransactionManager transactionManager(EntityManagerFactory entityManagerFactory) { + return new JpaTransactionManager(entityManagerFactory); + } +} \ No newline at end of file diff --git a/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/configuration/InMemoryBatchConfiguration.java b/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/configuration/InMemoryBatchConfiguration.java new file mode 100644 index 000000000..c638cd77b --- /dev/null +++ b/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/configuration/InMemoryBatchConfiguration.java @@ -0,0 +1,50 @@ +package uk.ac.ebi.eva.accession.core.configuration; + +import org.springframework.batch.core.configuration.JobRegistry; +import org.springframework.batch.core.configuration.support.MapJobRegistry; +import org.springframework.batch.core.configuration.support.ScopeConfiguration; +import org.springframework.batch.core.explore.JobExplorer; +import org.springframework.batch.core.launch.JobLauncher; +import org.springframework.batch.core.launch.support.TaskExecutorJobLauncher; +import org.springframework.batch.core.repository.JobRepository; +import org.springframework.batch.core.repository.support.ResourcelessJobRepository; +import org.springframework.batch.support.transaction.ResourcelessTransactionManager; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; +import org.springframework.transaction.PlatformTransactionManager; + +@Configuration +@Import(ScopeConfiguration.class) +public class InMemoryBatchConfiguration { + + public static final String BATCH_TRANSACTION_MANAGER = "batchTransactionManager"; + + @Bean(BATCH_TRANSACTION_MANAGER) + public PlatformTransactionManager batchTransactionManager() { + return new ResourcelessTransactionManager(); + } + + @Bean + public JobRepository jobRepository() { + return new ResourcelessJobRepository(); + } + + @Bean + public JobExplorer jobExplorer() { + return new NoOpJobExplorer(); + } + + @Bean + public JobRegistry jobRegistry() { + return new MapJobRegistry(); + } + + @Bean + public JobLauncher jobLauncher(JobRepository jobRepository) throws Exception { + TaskExecutorJobLauncher launcher = new TaskExecutorJobLauncher(); + launcher.setJobRepository(jobRepository); + launcher.afterPropertiesSet(); + return launcher; + } +} \ No newline at end of file diff --git a/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/configuration/NoOpJobExplorer.java b/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/configuration/NoOpJobExplorer.java new file mode 100644 index 000000000..99e3e4062 --- /dev/null +++ b/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/configuration/NoOpJobExplorer.java @@ -0,0 +1,65 @@ +package uk.ac.ebi.eva.accession.core.configuration; + +import org.springframework.batch.core.JobExecution; +import org.springframework.batch.core.JobInstance; +import org.springframework.batch.core.StepExecution; +import org.springframework.batch.core.explore.JobExplorer; +import org.springframework.lang.Nullable; + +import java.util.List; +import java.util.Set; + +public class NoOpJobExplorer implements JobExplorer { + @Override + public List getJobInstances(String jobName, int start, int count) { + return List.of(); + } + + @Nullable + @Override + public JobExecution getJobExecution(@Nullable Long executionId) { + return null; + } + + @Nullable + @Override + public StepExecution getStepExecution(@Nullable Long jobExecutionId, @Nullable Long stepExecutionId) { + return null; + } + + @Nullable + @Override + public JobInstance getJobInstance(@Nullable Long instanceId) { + return null; + } + + @Override + public List getJobExecutions(JobInstance jobInstance) { + return List.of(); + } + + @Override + public Set findRunningJobExecutions(@Nullable String jobName) { + return Set.of(); + } + + @Override + public List getJobNames() { + return List.of(); + } + + @Override + public List findJobInstancesByJobName(String jobName, int start, int count) { + return List.of(); + } + + @Override + public long getJobInstanceCount(@Nullable String jobName) { + return 0; + } + + @Override + public JobInstance getLastJobInstance(String jobName) { + return null; + } +} \ No newline at end of file diff --git a/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/runner/CommandLineRunnerUtils.java b/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/runner/CommandLineRunnerUtils.java deleted file mode 100644 index 589886bf4..000000000 --- a/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/runner/CommandLineRunnerUtils.java +++ /dev/null @@ -1,96 +0,0 @@ -package uk.ac.ebi.eva.accession.core.runner; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.batch.core.BatchStatus; -import org.springframework.batch.core.Entity; -import org.springframework.batch.core.JobExecution; -import org.springframework.batch.core.JobInstance; -import org.springframework.batch.core.JobParameter; -import org.springframework.batch.core.JobParameters; -import org.springframework.batch.core.JobParametersBuilder; -import org.springframework.batch.core.explore.JobExplorer; -import org.springframework.batch.core.launch.NoSuchJobException; -import org.springframework.batch.core.repository.JobRepository; -import uk.ac.ebi.eva.commons.batch.exception.NoPreviousJobExecutionException; -import uk.ac.ebi.eva.commons.batch.job.JobStatusManager; - -import java.util.Comparator; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -public class CommandLineRunnerUtils { - - private static final Logger logger = LoggerFactory.getLogger(CommandLineRunnerUtils.class); - - private static final String RUN_ID_PARAMETER_NAME = "run.id"; - - public static JobParameters addRunIDToJobParameters(String jobName, JobExplorer jobExplorer, - JobParameters jobParameters) { - JobExecution lastJobExecution = getLastJobExecution(jobName, jobExplorer, jobParameters); - if (lastJobExecution != null) { - Long runIdParameterFromLastExecution = lastJobExecution.getJobParameters() - .getLong(RUN_ID_PARAMETER_NAME); - if (runIdParameterFromLastExecution != 0 && lastJobExecution.getStatus() == BatchStatus.FAILED) { - // Spring Batch 4 uses all job parameters (including run.id) to detect previous instances of a job - see https://github.com/spring-projects/spring-boot/blob/86fb39d5c5f474fe3544159270d4c4e2d01d43ef/spring-boot-project/spring-boot-autoconfigure/src/main/java/org/springframework/boot/autoconfigure/batch/JobLauncherCommandLineRunner.java#L222 - // as opposed to Spring 3 which uses only job name - see https://github.com/spring-projects/spring-boot/blob/541890f0e003a6e346f2234102c97105ab1292ee/spring-boot-autoconfigure/src/main/java/org/springframework/boot/autoconfigure/batch/JobLauncherCommandLineRunner.java#L131 - // Therefore, run.id needs to be supplied for failed jobs in order for the job to be detected and resumed - see https://stackoverflow.com/a/59198742/2601814 - return new JobParametersBuilder(jobParameters) - .addLong(RUN_ID_PARAMETER_NAME, runIdParameterFromLastExecution).toJobParameters(); - } - } - - return jobParameters; - } - - public static JobExecution getLastJobExecution(String jobName, JobExplorer jobExplorer, - JobParameters previousJobParameters) { - long previousJobInstanceCount = getPreviousJobInstanceCount(jobName, jobExplorer); - - List jobInstanceList = jobExplorer.getJobInstances(jobName, 0, (int) previousJobInstanceCount); - List matchingJobExecutions; - - for (JobInstance jobInstance : jobInstanceList) { - matchingJobExecutions = jobExplorer.getJobExecutions(jobInstance); - for (JobExecution jobExecution : matchingJobExecutions) { - if (areParametersEquivalentExceptRunId(jobExecution.getJobParameters(), previousJobParameters)) { - return matchingJobExecutions.stream().max(Comparator.comparingLong(Entity::getId)).get(); - } - } - } - return null; - } - - private static long getPreviousJobInstanceCount(String jobName, JobExplorer jobExplorer) { - try { - return jobExplorer.getJobInstanceCount(jobName); - } catch (NoSuchJobException ex) { - return 0; - } - } - - private static boolean areParametersEquivalentExceptRunId(JobParameters parameters1, JobParameters parameters2) { - Map> firstJobParameterMap = new HashMap<>(parameters1.getParameters()); - Map> secondJobParameterMap = new HashMap<>(parameters2.getParameters()); - if (parameters1.getParameters().containsKey(RUN_ID_PARAMETER_NAME)) { - firstJobParameterMap.remove(RUN_ID_PARAMETER_NAME); - } - if (parameters2.getParameters().containsKey(RUN_ID_PARAMETER_NAME)) { - secondJobParameterMap.remove(RUN_ID_PARAMETER_NAME); - } - return firstJobParameterMap.equals(secondJobParameterMap); - } - - public static void markPreviousJobAsFailed(String jobName, JobRepository jobRepository, JobParameters jobParameters) - throws NoPreviousJobExecutionException { - logger.info("Force restartPreviousExecution of job '" + jobName + "' with parameters: " + jobParameters); - try { - JobStatusManager.markLastJobAsFailed(jobRepository, jobName, jobParameters); - } catch (NoPreviousJobExecutionException ex) { - //Be forgiving if a previous job does not exist: just print a warning message - logger.warn(ex.getMessage()); - } - } - -} diff --git a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/batch/io/ClusteredVariantDeprecationWriterTest.java b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/batch/io/ClusteredVariantDeprecationWriterTest.java index 0e864cedb..77e0c1ce7 100644 --- a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/batch/io/ClusteredVariantDeprecationWriterTest.java +++ b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/batch/io/ClusteredVariantDeprecationWriterTest.java @@ -22,12 +22,12 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.batch.item.Chunk; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.ampt2d.commons.accession.hashing.SHA1HashingFunction; +import uk.ac.ebi.eva.accession.core.configuration.ContiguousIdBlocksDataSourceConfiguration; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.ClusteredVariantAccessioningConfiguration; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.MongoConfiguration; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.SubmittedVariantAccessioningConfiguration; @@ -53,9 +53,9 @@ @ExtendWith(SpringExtension.class) @TestPropertySource("classpath:ss-deprecation-test.properties") -@EnableAutoConfiguration @ContextConfiguration(classes = {MongoConfiguration.class, MongoTestConfiguration.class, - SubmittedVariantAccessioningConfiguration.class, ClusteredVariantAccessioningConfiguration.class}) + SubmittedVariantAccessioningConfiguration.class, ClusteredVariantAccessioningConfiguration.class, + ContiguousIdBlocksDataSourceConfiguration.class}) public class ClusteredVariantDeprecationWriterTest extends MongoTestContainerHelper { private static final String ASSEMBLY = "GCA_000000001.1"; diff --git a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/batch/io/SubmittedVariantDeprecationWriterTest.java b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/batch/io/SubmittedVariantDeprecationWriterTest.java index 8179d4a33..d7c60a26f 100644 --- a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/batch/io/SubmittedVariantDeprecationWriterTest.java +++ b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/batch/io/SubmittedVariantDeprecationWriterTest.java @@ -22,12 +22,12 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.batch.item.Chunk; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.ampt2d.commons.accession.hashing.SHA1HashingFunction; +import uk.ac.ebi.eva.accession.core.configuration.ContiguousIdBlocksDataSourceConfiguration; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.ClusteredVariantAccessioningConfiguration; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.MongoConfiguration; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.SubmittedVariantAccessioningConfiguration; @@ -56,13 +56,11 @@ @ExtendWith(SpringExtension.class) @TestPropertySource("classpath:ss-deprecation-test.properties") -@EnableAutoConfiguration @ContextConfiguration(classes = {MongoConfiguration.class, MongoTestConfiguration.class, - SubmittedVariantAccessioningConfiguration.class, ClusteredVariantAccessioningConfiguration.class}) + SubmittedVariantAccessioningConfiguration.class, ClusteredVariantAccessioningConfiguration.class, + ContiguousIdBlocksDataSourceConfiguration.class}) public class SubmittedVariantDeprecationWriterTest extends MongoTestContainerHelper { - private static final String TEST_DB = "sve-deprecation-test"; - private static final String ASSEMBLY = "GCA_000000001.1"; private static final int TAXONOMY = 60711; diff --git a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/model/eva/ClusteredVariantEntityIntegrationTest.java b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/model/eva/ClusteredVariantEntityIntegrationTest.java index 7d535412e..eafc5dc00 100644 --- a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/model/eva/ClusteredVariantEntityIntegrationTest.java +++ b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/model/eva/ClusteredVariantEntityIntegrationTest.java @@ -20,7 +20,6 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.data.mongodb.core.query.Query; import org.springframework.test.context.ContextConfiguration; @@ -41,7 +40,6 @@ import static org.junit.jupiter.api.Assertions.assertEquals; @ExtendWith(SpringExtension.class) -@EnableAutoConfiguration @ContextConfiguration(classes = {MongoConfiguration.class, MongoTestConfiguration.class}) @TestPropertySource("classpath:test-model.properties") public class ClusteredVariantEntityIntegrationTest extends MongoTestContainerHelper { diff --git a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/model/eva/SubmittedVariantEntityIntegrationTest.java b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/model/eva/SubmittedVariantEntityIntegrationTest.java index 92ed34acd..e8ea13b62 100644 --- a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/model/eva/SubmittedVariantEntityIntegrationTest.java +++ b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/model/eva/SubmittedVariantEntityIntegrationTest.java @@ -20,7 +20,6 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.data.mongodb.core.query.Query; import org.springframework.test.context.ContextConfiguration; @@ -40,7 +39,6 @@ import static org.junit.jupiter.api.Assertions.assertEquals; @ExtendWith(SpringExtension.class) -@EnableAutoConfiguration @ContextConfiguration(classes = {MongoConfiguration.class, MongoTestConfiguration.class}) @TestPropertySource("classpath:test-model.properties") public class SubmittedVariantEntityIntegrationTest extends MongoTestContainerHelper { diff --git a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/repository/nonhuman/eva/SubmittedVariantAccessioningRepositoryTest.java b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/repository/nonhuman/eva/SubmittedVariantAccessioningRepositoryTest.java index bb6d40b05..80137fd19 100644 --- a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/repository/nonhuman/eva/SubmittedVariantAccessioningRepositoryTest.java +++ b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/repository/nonhuman/eva/SubmittedVariantAccessioningRepositoryTest.java @@ -26,10 +26,10 @@ import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.ampt2d.commons.accession.persistence.models.AccessionProjection; import uk.ac.ebi.ampt2d.commons.accession.persistence.mongodb.document.AccessionedDocument; +import uk.ac.ebi.eva.accession.core.configuration.ContiguousIdBlocksDataSourceConfiguration; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.SubmittedVariantAccessioningConfiguration; import uk.ac.ebi.eva.accession.core.model.SubmittedVariant; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; -import uk.ac.ebi.eva.accession.core.test.configuration.JPATestConfiguration; import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; @@ -47,7 +47,7 @@ @ExtendWith(SpringExtension.class) @TestPropertySource("classpath:ss-accession-test.properties") @ContextConfiguration(classes = {SubmittedVariantAccessioningConfiguration.class, MongoTestConfiguration.class, - JPATestConfiguration.class}) + ContiguousIdBlocksDataSourceConfiguration.class}) public class SubmittedVariantAccessioningRepositoryTest extends MongoTestContainerHelper { private static final Long CLUSTERED_VARIANT = null; diff --git a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/service/human/dbsnp/HumanDbsnpClusteredVariantAccessioningServiceTest.java b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/service/human/dbsnp/HumanDbsnpClusteredVariantAccessioningServiceTest.java index 30682e090..ce619d43a 100644 --- a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/service/human/dbsnp/HumanDbsnpClusteredVariantAccessioningServiceTest.java +++ b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/service/human/dbsnp/HumanDbsnpClusteredVariantAccessioningServiceTest.java @@ -28,9 +28,9 @@ import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.ampt2d.commons.accession.core.models.AccessionWrapper; +import uk.ac.ebi.eva.accession.core.configuration.ContiguousIdBlocksDataSourceConfiguration; import uk.ac.ebi.eva.accession.core.model.ClusteredVariant; import uk.ac.ebi.eva.accession.core.model.IClusteredVariant; -import uk.ac.ebi.eva.accession.core.test.configuration.JPATestConfiguration; import uk.ac.ebi.eva.accession.core.test.configuration.human.MongoHumanTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.core.utils.MongoTestDataLoader; @@ -45,7 +45,7 @@ @ExtendWith(SpringExtension.class) @TestPropertySource("classpath:ss-accession-test.properties") -@ContextConfiguration(classes = {MongoHumanTestConfiguration.class, JPATestConfiguration.class}) +@ContextConfiguration(classes = {MongoHumanTestConfiguration.class, ContiguousIdBlocksDataSourceConfiguration.class}) public class HumanDbsnpClusteredVariantAccessioningServiceTest extends MongoTestContainerHelper { private static final Long HUMAN_ACTIVE_RS_ID_1 = 1118L; diff --git a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/service/nonhuman/SubmittedVariantAccessioningServiceTest.java b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/service/nonhuman/SubmittedVariantAccessioningServiceTest.java index 9a1b6c91a..6229deb42 100644 --- a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/service/nonhuman/SubmittedVariantAccessioningServiceTest.java +++ b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/service/nonhuman/SubmittedVariantAccessioningServiceTest.java @@ -22,9 +22,12 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.io.ClassPathResource; import org.springframework.core.io.ResourceLoader; import org.springframework.data.mongodb.MongoDatabaseFactory; import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.jdbc.datasource.init.ResourceDatabasePopulator; +import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit.jupiter.SpringExtension; @@ -39,6 +42,7 @@ import uk.ac.ebi.ampt2d.commons.accession.core.models.GetOrCreateAccessionWrapper; import uk.ac.ebi.ampt2d.commons.accession.core.models.IEvent; import uk.ac.ebi.ampt2d.commons.accession.hashing.SHA1HashingFunction; +import uk.ac.ebi.eva.accession.core.configuration.ContiguousIdBlocksDataSourceConfiguration; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.SubmittedVariantAccessioningConfiguration; import uk.ac.ebi.eva.accession.core.generators.DbsnpMonotonicAccessionGenerator; import uk.ac.ebi.eva.accession.core.model.ISubmittedVariant; @@ -48,12 +52,15 @@ import uk.ac.ebi.eva.accession.core.service.nonhuman.dbsnp.DbsnpSubmittedVariantMonotonicAccessioningService; import uk.ac.ebi.eva.accession.core.service.nonhuman.eva.SubmittedVariantInactiveService; import uk.ac.ebi.eva.accession.core.summary.SubmittedVariantSummaryFunction; -import uk.ac.ebi.eva.accession.core.test.configuration.JPATestConfiguration; import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.core.utils.MongoTestDataLoader; import uk.ac.ebi.eva.commons.core.models.contigalias.ContigNamingConvention; +import javax.sql.DataSource; +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; import java.time.LocalDateTime; import java.time.Month; import java.util.ArrayList; @@ -76,7 +83,8 @@ @ExtendWith(SpringExtension.class) @TestPropertySource("classpath:ss-accession-test.properties") @ContextConfiguration(classes = {SubmittedVariantAccessioningConfiguration.class, MongoTestConfiguration.class, - JPATestConfiguration.class}) + ContiguousIdBlocksDataSourceConfiguration.class}) +@DirtiesContext(classMode = DirtiesContext.ClassMode.BEFORE_EACH_TEST_METHOD) public class SubmittedVariantAccessioningServiceTest extends MongoTestContainerHelper { private static String TEST_APPLICATION_INSTANCE_ID = "test-application-instance-id"; @@ -130,6 +138,9 @@ public class SubmittedVariantAccessioningServiceTest extends MongoTestContainerH @Autowired private Long accessioningMonotonicInitSs; + @Autowired + private DataSource dataSource; + @Autowired private MongoDatabaseFactory mongoDbFactory; @@ -140,7 +151,7 @@ public class SubmittedVariantAccessioningServiceTest extends MongoTestContainerH private MongoTemplate mongoTemplate; @BeforeEach - public void setUp() { + public void setUp() throws SQLException { mongoTemplate.getDb().drop(); submittedVariant = new SubmittedVariant("GCA_000003055.3", 9913, PROJECT, "21", 20800319, "C", "T", CLUSTERED_VARIANT, DEFAULT_SUPPORTED_BY_EVIDENCE, @@ -160,6 +171,14 @@ public void setUp() { submittedVariantModified = new SubmittedVariant("GCA_000003055.3", 9913, PROJECT, "21", 20800319, "C", "TCTC", CLUSTERED_VARIANT, DEFAULT_SUPPORTED_BY_EVIDENCE, DEFAULT_ASSEMBLY_MATCH, true, DEFAULT_VALIDATED, null); + + try (Connection conn = dataSource.getConnection(); + Statement stmt = conn.createStatement()) { + stmt.execute("DROP TABLE IF EXISTS contiguous_id_blocks"); + } + ResourceDatabasePopulator populator = new ResourceDatabasePopulator(); + populator.addScript(new ClassPathResource("test-data/contiguous_id_blocks_schema.sql")); + populator.execute(dataSource); } @AfterEach @@ -169,6 +188,7 @@ public void tearDown() { @Test public void sameAccessionsAreReturnedForIdenticalVariants() throws AccessionCouldNotBeGeneratedException { + List variants = Arrays.asList( new SubmittedVariant("assembly", 1111, "project", "contig_1", 100, "ref", "alt", CLUSTERED_VARIANT, DEFAULT_SUPPORTED_BY_EVIDENCE, DEFAULT_ASSEMBLY_MATCH, ALLELES_MATCH, diff --git a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/test/configuration/JPATestConfiguration.java b/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/test/configuration/JPATestConfiguration.java deleted file mode 100644 index 66a846d06..000000000 --- a/eva-accession-core/src/test/java/uk/ac/ebi/eva/accession/core/test/configuration/JPATestConfiguration.java +++ /dev/null @@ -1,41 +0,0 @@ -package uk.ac.ebi.eva.accession.core.test.configuration; - -import jakarta.persistence.EntityManagerFactory; -import org.springframework.context.annotation.Bean; -import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseBuilder; -import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseType; -import org.springframework.orm.jpa.JpaTransactionManager; -import org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean; -import org.springframework.orm.jpa.vendor.HibernateJpaVendorAdapter; -import org.springframework.transaction.PlatformTransactionManager; - -import javax.sql.DataSource; - -public class JPATestConfiguration { - @Bean - public DataSource dataSource() { - return new EmbeddedDatabaseBuilder() - .setType(EmbeddedDatabaseType.HSQL) - .build(); - } - - @Bean - public LocalContainerEntityManagerFactoryBean entityManagerFactory(DataSource dataSource) { - LocalContainerEntityManagerFactoryBean em = new LocalContainerEntityManagerFactoryBean(); - em.setDataSource(dataSource); - // Package where ContiguousIdBlockEntity (the JPA entity) lives - em.setPackagesToScan( - "uk.ac.ebi.ampt2d.commons.accession.persistence.jpa.monotonic.entities" - ); - HibernateJpaVendorAdapter vendorAdapter = new HibernateJpaVendorAdapter(); - vendorAdapter.setGenerateDdl(true); // auto-creates the schema in H2 - vendorAdapter.setShowSql(false); - em.setJpaVendorAdapter(vendorAdapter); - return em; - } - - @Bean - public PlatformTransactionManager transactionManager(EntityManagerFactory entityManagerFactory) { - return new JpaTransactionManager(entityManagerFactory); - } -} diff --git a/eva-accession-core/src/test/resources/properties/ss-accession-test.properties b/eva-accession-core/src/test/resources/properties/ss-accession-test.properties index b4151457a..5c9e591cc 100644 --- a/eva-accession-core/src/test/resources/properties/ss-accession-test.properties +++ b/eva-accession-core/src/test/resources/properties/ss-accession-test.properties @@ -8,6 +8,11 @@ accessioning.monotonic.test-rs.blockSize=100000 accessioning.monotonic.test-rs.blockStartValue=3000000000 accessioning.monotonic.test-rs.nextBlockInterval=1000000000 +continuous.id.blocks.datasource.driver-class-name=org.hsqldb.jdbc.JDBCDriver +continuous.id.blocks.datasource.jdbc-url=jdbc:hsqldb:mem:db;sql.syntax_pgs=true;DB_CLOSE_DELAY=-1 +continuous.id.blocks.datasource.username=SA +continuous.id.blocks.datasource.password= + spring.data.mongodb.database=submitted-variants-test # This symbolic variable is required because # this will be replaced with a default value "localhost" when run locally diff --git a/eva-accession-core/src/test/resources/properties/ss-deprecation-test.properties b/eva-accession-core/src/test/resources/properties/ss-deprecation-test.properties index 50d8e3cd5..f9a467bb5 100644 --- a/eva-accession-core/src/test/resources/properties/ss-deprecation-test.properties +++ b/eva-accession-core/src/test/resources/properties/ss-deprecation-test.properties @@ -9,6 +9,11 @@ accessioning.monotonic.test-rs.blockSize=100000 accessioning.monotonic.test-rs.blockStartValue=4 accessioning.monotonic.test-rs.nextBlockInterval=1000000000 +continuous.id.blocks.datasource.driver-class-name=org.hsqldb.jdbc.JDBCDriver +continuous.id.blocks.datasource.jdbc-url=jdbc:hsqldb:mem:db;sql.syntax_pgs=true;DB_CLOSE_DELAY=-1 +continuous.id.blocks.datasource.username=SA +continuous.id.blocks.datasource.password= + spring.data.mongodb.database=sve-deprecation-test # This symbolic variable is required because # this will be replaced with a default value "localhost" when run locally diff --git a/eva-accession-core/src/test/resources/test-data/contiguous_id_blocks_schema.sql b/eva-accession-core/src/test/resources/test-data/contiguous_id_blocks_schema.sql new file mode 100644 index 000000000..32dfe369d --- /dev/null +++ b/eva-accession-core/src/test/resources/test-data/contiguous_id_blocks_schema.sql @@ -0,0 +1,14 @@ +CREATE SEQUENCE IF NOT EXISTS contiguous_id_blocks_seq + START WITH 1 + INCREMENT BY 50; + +CREATE TABLE IF NOT EXISTS contiguous_id_blocks ( + id bigint not NULL, + application_instance_id varchar(255) not NULL, + category_id varchar(255) not NULL, + first_value bigint not NULL, + last_committed bigint not NULL, + last_value bigint not NULL, + reserved boolean NOT NULL, + last_updated_timestamp timestamp NOT NULL +); diff --git a/eva-accession-deprecate/pom.xml b/eva-accession-deprecate/pom.xml index 71cf08411..da20d7e0a 100644 --- a/eva-accession-deprecate/pom.xml +++ b/eva-accession-deprecate/pom.xml @@ -28,6 +28,11 @@ xstream + + org.postgresql + postgresql + + uk.ac.ebi.eva eva-accession-core @@ -36,6 +41,12 @@ test + + org.hsqldb + hsqldb + test + + org.springframework.boot spring-boot-testcontainers diff --git a/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/Application.java b/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/Application.java index 1a67c884d..498c87682 100644 --- a/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/Application.java +++ b/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/Application.java @@ -15,13 +15,26 @@ */ package uk.ac.ebi.eva.accession.deprecate; -import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.autoconfigure.batch.BatchAutoConfiguration; +import org.springframework.boot.autoconfigure.data.mongo.MongoDataAutoConfiguration; +import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration; +import org.springframework.boot.autoconfigure.jdbc.DataSourceTransactionManagerAutoConfiguration; +import org.springframework.boot.autoconfigure.orm.jpa.HibernateJpaAutoConfiguration; import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.context.annotation.Import; +import uk.ac.ebi.eva.accession.core.configuration.ContiguousIdBlocksDataSourceConfiguration; +import uk.ac.ebi.eva.accession.core.configuration.InMemoryBatchConfiguration; -@EnableBatchProcessing -@SpringBootApplication +@SpringBootApplication(exclude = { + MongoDataAutoConfiguration.class, + DataSourceAutoConfiguration.class, + DataSourceTransactionManagerAutoConfiguration.class, + HibernateJpaAutoConfiguration.class, + BatchAutoConfiguration.class +}) +@Import({InMemoryBatchConfiguration.class, ContiguousIdBlocksDataSourceConfiguration.class}) public class Application { public static void main(String[] args) { diff --git a/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/batch/jobs/DeprecateStudySubmittedVariantsJobConfiguration.java b/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/batch/jobs/DeprecateStudySubmittedVariantsJobConfiguration.java index ebe7fa92b..1e23f8e17 100644 --- a/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/batch/jobs/DeprecateStudySubmittedVariantsJobConfiguration.java +++ b/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/batch/jobs/DeprecateStudySubmittedVariantsJobConfiguration.java @@ -17,9 +17,7 @@ import org.springframework.batch.core.Job; import org.springframework.batch.core.Step; -import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.batch.core.job.builder.JobBuilder; -import org.springframework.batch.core.launch.support.RunIdIncrementer; import org.springframework.batch.core.repository.JobRepository; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; @@ -28,7 +26,6 @@ import uk.ac.ebi.eva.accession.deprecate.configuration.BeanNames; @Configuration -@EnableBatchProcessing public class DeprecateStudySubmittedVariantsJobConfiguration { @Autowired @@ -38,7 +35,6 @@ public class DeprecateStudySubmittedVariantsJobConfiguration { @Bean(BeanNames.DEPRECATE_STUDY_SUBMITTED_VARIANTS_JOB) public Job accessionReleaseJob(JobRepository jobRepository) { return new JobBuilder(BeanNames.DEPRECATE_STUDY_SUBMITTED_VARIANTS_JOB, jobRepository) - .incrementer(new RunIdIncrementer()) .start(deprecateStudySubmittedVariantsStep) .build(); } diff --git a/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/batch/jobs/DeprecateSubmittedVariantsFromFileJobConfiguration.java b/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/batch/jobs/DeprecateSubmittedVariantsFromFileJobConfiguration.java index 05958a588..14b09a85c 100644 --- a/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/batch/jobs/DeprecateSubmittedVariantsFromFileJobConfiguration.java +++ b/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/batch/jobs/DeprecateSubmittedVariantsFromFileJobConfiguration.java @@ -17,9 +17,7 @@ import org.springframework.batch.core.Job; import org.springframework.batch.core.Step; -import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.batch.core.job.builder.JobBuilder; -import org.springframework.batch.core.launch.support.RunIdIncrementer; import org.springframework.batch.core.repository.JobRepository; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; @@ -28,7 +26,6 @@ import uk.ac.ebi.eva.accession.deprecate.configuration.BeanNames; @Configuration -@EnableBatchProcessing public class DeprecateSubmittedVariantsFromFileJobConfiguration { @Autowired @@ -38,7 +35,6 @@ public class DeprecateSubmittedVariantsFromFileJobConfiguration { @Bean(BeanNames.DEPRECATE_SUBMITTED_VARIANTS_FROM_FILE_JOB) public Job deprecateStudySubmittedVariantsFromFileJob(JobRepository jobRepository) { return new JobBuilder(BeanNames.DEPRECATE_SUBMITTED_VARIANTS_FROM_FILE_JOB, jobRepository) - .incrementer(new RunIdIncrementer()) .start(deprecateSubmittedVariantsFromFileStep) .build(); } diff --git a/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/batch/steps/DeprecateStudySubmittedVariantsStepConfiguration.java b/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/batch/steps/DeprecateStudySubmittedVariantsStepConfiguration.java index 37261c120..3fa8ec836 100644 --- a/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/batch/steps/DeprecateStudySubmittedVariantsStepConfiguration.java +++ b/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/batch/steps/DeprecateStudySubmittedVariantsStepConfiguration.java @@ -17,7 +17,6 @@ import org.springframework.batch.core.Step; import org.springframework.batch.core.StepExecutionListener; -import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.batch.core.repository.JobRepository; import org.springframework.batch.core.step.builder.StepBuilder; import org.springframework.batch.core.step.tasklet.TaskletStep; @@ -32,8 +31,9 @@ import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; import uk.ac.ebi.eva.accession.deprecate.configuration.BeanNames; +import static uk.ac.ebi.eva.accession.core.configuration.InMemoryBatchConfiguration.BATCH_TRANSACTION_MANAGER; + @Configuration -@EnableBatchProcessing public class DeprecateStudySubmittedVariantsStepConfiguration { @Autowired @@ -50,6 +50,7 @@ public class DeprecateStudySubmittedVariantsStepConfiguration { @Bean(BeanNames.DEPRECATE_STUDY_SUBMITTED_VARIANTS_STEP) public Step deprecateClusteredVariantsStep(JobRepository jobRepository, + @Qualifier(BATCH_TRANSACTION_MANAGER) PlatformTransactionManager transactionManager, SimpleCompletionPolicy chunkSizeCompletionPolicy) { TaskletStep step = new StepBuilder(BeanNames.DEPRECATE_STUDY_SUBMITTED_VARIANTS_STEP, jobRepository) diff --git a/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/batch/steps/DeprecateSubmittedVariantsFromFileStepConfiguration.java b/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/batch/steps/DeprecateSubmittedVariantsFromFileStepConfiguration.java index aad2dfedd..c07280dac 100644 --- a/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/batch/steps/DeprecateSubmittedVariantsFromFileStepConfiguration.java +++ b/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/batch/steps/DeprecateSubmittedVariantsFromFileStepConfiguration.java @@ -17,7 +17,6 @@ import org.springframework.batch.core.Step; import org.springframework.batch.core.StepExecutionListener; -import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.batch.core.repository.JobRepository; import org.springframework.batch.core.step.builder.StepBuilder; import org.springframework.batch.core.step.tasklet.TaskletStep; @@ -32,8 +31,9 @@ import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; import uk.ac.ebi.eva.accession.deprecate.configuration.BeanNames; +import static uk.ac.ebi.eva.accession.core.configuration.InMemoryBatchConfiguration.BATCH_TRANSACTION_MANAGER; + @Configuration -@EnableBatchProcessing public class DeprecateSubmittedVariantsFromFileStepConfiguration { @Autowired @@ -50,6 +50,7 @@ public class DeprecateSubmittedVariantsFromFileStepConfiguration { @Bean(BeanNames.DEPRECATE_SUBMITTED_VARIANTS_FROM_FILE_STEP) public Step deprecateSubmittedVariantsFromFileStep(JobRepository jobRepository, + @Qualifier(BATCH_TRANSACTION_MANAGER) PlatformTransactionManager transactionManager, SimpleCompletionPolicy chunkSizeCompletionPolicy) { TaskletStep step = new StepBuilder(BeanNames.DEPRECATE_SUBMITTED_VARIANTS_FROM_FILE_STEP, jobRepository) diff --git a/eva-accession-deprecate/src/main/resources/application.properties b/eva-accession-deprecate/src/main/resources/application.properties index 798e22105..15712493e 100644 --- a/eva-accession-deprecate/src/main/resources/application.properties +++ b/eva-accession-deprecate/src/main/resources/application.properties @@ -22,12 +22,6 @@ spring.data.mongodb.password= spring.data.mongodb.authentication-database=admin mongodb.read-preference=primary -spring.datasource.driver-class-name=org.postgresql.Driver -spring.datasource.url= -spring.datasource.username= -spring.datasource.password= -spring.datasource.tomcat.max-active=3 - #See https://github.com/spring-projects/spring-boot/wiki/Spring-Boot-2.1-Release-Notes#bean-overriding spring.main.allow-bean-definition-overriding=true #As this is a spring batch application, disable the embedded tomcat. This is the new way to do that for spring 2. diff --git a/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/batch/io/StudySubmittedVariantsReaderTest.java b/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/batch/io/StudySubmittedVariantsReaderTest.java index ea6b98198..e552b02b1 100644 --- a/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/batch/io/StudySubmittedVariantsReaderTest.java +++ b/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/batch/io/StudySubmittedVariantsReaderTest.java @@ -27,8 +27,8 @@ import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.MongoConfiguration; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; -import uk.ac.ebi.eva.accession.deprecate.test.configuration.MongoTestConfiguration; import java.util.ArrayList; import java.util.List; diff --git a/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/configuration/jobs/DeprecateStudySubmittedVariantsJobConfigurationTest.java b/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/configuration/jobs/DeprecateStudySubmittedVariantsJobConfigurationTest.java index 2c44eb795..d2b3d6cd7 100644 --- a/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/configuration/jobs/DeprecateStudySubmittedVariantsJobConfigurationTest.java +++ b/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/configuration/jobs/DeprecateStudySubmittedVariantsJobConfigurationTest.java @@ -29,16 +29,14 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.data.mongodb.core.MongoTemplate; -import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit.jupiter.SpringExtension; -import uk.ac.ebi.eva.accession.core.runner.CommandLineRunnerUtils; +import uk.ac.ebi.eva.accession.core.configuration.ContiguousIdBlocksDataSourceConfiguration; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.deprecate.MongoTestDatabaseSetup; -import uk.ac.ebi.eva.accession.deprecate.test.configuration.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.accession.deprecate.test.configuration.BatchTestConfiguration; -import uk.ac.ebi.eva.accession.deprecate.test.configuration.MongoTestConfiguration; import java.util.Collection; import java.util.Collections; @@ -46,14 +44,12 @@ import java.util.stream.Collectors; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotEquals; -import static uk.ac.ebi.eva.accession.deprecate.configuration.BeanNames.DEPRECATE_STUDY_SUBMITTED_VARIANTS_JOB; import static uk.ac.ebi.eva.accession.deprecate.configuration.BeanNames.DEPRECATE_STUDY_SUBMITTED_VARIANTS_STEP; import static uk.ac.ebi.eva.accession.deprecate.test.configuration.BatchTestConfiguration.JOB_LAUNCHER_FROM_MONGO; @ExtendWith(SpringExtension.class) @ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class, - BatchJobRepositoryTestConfiguration.class}) + ContiguousIdBlocksDataSourceConfiguration.class}) @TestPropertySource("classpath:study-submitted-variants-test.properties") public class DeprecateStudySubmittedVariantsJobConfigurationTest extends MongoTestContainerHelper { @Autowired @@ -101,31 +97,4 @@ private void assertStepsExecuted(List expectedSteps, JobExecution jobExecution) List steps = stepExecutions.stream().map(StepExecution::getStepName).collect(Collectors.toList()); assertEquals(expectedSteps, steps); } - - @Test - @DirtiesContext - public void restartCompletedJobThatIsAlreadyInTheRepository() throws Exception { - JobExecution jobExecution = jobLauncherTestUtilsFromMongo.launchJob(); - - List expectedSteps = Collections.singletonList(DEPRECATE_STUDY_SUBMITTED_VARIANTS_STEP); - assertStepsExecuted(expectedSteps, jobExecution); - assertEquals(BatchStatus.COMPLETED, jobExecution.getStatus()); - MongoTestDatabaseSetup.assertPostDeprecationDatabaseState(this.mongoTemplate); - - long instanceIdFirstJob = CommandLineRunnerUtils.getLastJobExecution(DEPRECATE_STUDY_SUBMITTED_VARIANTS_JOB, - jobExplorer, - jobExecution.getJobParameters()) - .getJobInstance().getInstanceId(); - - jobExecution = jobLauncherTestUtilsFromMongo.launchJob(); - expectedSteps = Collections.singletonList(DEPRECATE_STUDY_SUBMITTED_VARIANTS_STEP); - assertStepsExecuted(expectedSteps, jobExecution); - assertEquals(BatchStatus.COMPLETED, jobExecution.getStatus()); - long instanceIdSecondJob = CommandLineRunnerUtils.getLastJobExecution(DEPRECATE_STUDY_SUBMITTED_VARIANTS_JOB, - jobExplorer, - jobExecution.getJobParameters()) - .getJobInstance().getInstanceId(); - assertNotEquals(instanceIdSecondJob, instanceIdFirstJob); - MongoTestDatabaseSetup.assertPostDeprecationDatabaseState(this.mongoTemplate); - } } diff --git a/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/configuration/jobs/DeprecateSubmittedVariantsFromFileJobConfigurationTest.java b/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/configuration/jobs/DeprecateSubmittedVariantsFromFileJobConfigurationTest.java index 693ff64f1..819df4c0b 100644 --- a/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/configuration/jobs/DeprecateSubmittedVariantsFromFileJobConfigurationTest.java +++ b/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/configuration/jobs/DeprecateSubmittedVariantsFromFileJobConfigurationTest.java @@ -29,16 +29,14 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.data.mongodb.core.MongoTemplate; -import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit.jupiter.SpringExtension; -import uk.ac.ebi.eva.accession.core.runner.CommandLineRunnerUtils; +import uk.ac.ebi.eva.accession.core.configuration.ContiguousIdBlocksDataSourceConfiguration; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.deprecate.MongoTestDatabaseSetup; -import uk.ac.ebi.eva.accession.deprecate.test.configuration.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.accession.deprecate.test.configuration.BatchTestConfiguration; -import uk.ac.ebi.eva.accession.deprecate.test.configuration.MongoTestConfiguration; import java.util.Collection; import java.util.Collections; @@ -46,14 +44,12 @@ import java.util.stream.Collectors; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotEquals; -import static uk.ac.ebi.eva.accession.deprecate.configuration.BeanNames.DEPRECATE_SUBMITTED_VARIANTS_FROM_FILE_JOB; import static uk.ac.ebi.eva.accession.deprecate.configuration.BeanNames.DEPRECATE_SUBMITTED_VARIANTS_FROM_FILE_STEP; import static uk.ac.ebi.eva.accession.deprecate.test.configuration.BatchTestConfiguration.JOB_LAUNCHER_FROM_FILE; @ExtendWith(SpringExtension.class) @ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class, - BatchJobRepositoryTestConfiguration.class}) + ContiguousIdBlocksDataSourceConfiguration.class}) @TestPropertySource("classpath:deprecate-submitted-variants-from-file-test.properties") public class DeprecateSubmittedVariantsFromFileJobConfigurationTest extends MongoTestContainerHelper { @Autowired @@ -101,31 +97,4 @@ private void assertStepsExecuted(List expectedSteps, JobExecution jobExecution) List steps = stepExecutions.stream().map(StepExecution::getStepName).collect(Collectors.toList()); assertEquals(expectedSteps, steps); } - - @Test - @DirtiesContext - public void restartCompletedJobThatIsAlreadyInTheRepository() throws Exception { - JobExecution jobExecution = jobLauncherTestUtilsFromFile.launchJob(); - - List expectedSteps = Collections.singletonList(DEPRECATE_SUBMITTED_VARIANTS_FROM_FILE_STEP); - assertStepsExecuted(expectedSteps, jobExecution); - assertEquals(BatchStatus.COMPLETED, jobExecution.getStatus()); - MongoTestDatabaseSetup.assertPostDeprecationDatabaseState(this.mongoTemplate); - - long instanceIdFirstJob = CommandLineRunnerUtils.getLastJobExecution(DEPRECATE_SUBMITTED_VARIANTS_FROM_FILE_JOB, - jobExplorer, - jobExecution.getJobParameters()) - .getJobInstance().getInstanceId(); - - jobExecution = jobLauncherTestUtilsFromFile.launchJob(); - expectedSteps = Collections.singletonList(DEPRECATE_SUBMITTED_VARIANTS_FROM_FILE_STEP); - assertStepsExecuted(expectedSteps, jobExecution); - assertEquals(BatchStatus.COMPLETED, jobExecution.getStatus()); - long instanceIdSecondJob = CommandLineRunnerUtils.getLastJobExecution(DEPRECATE_SUBMITTED_VARIANTS_FROM_FILE_JOB, - jobExplorer, - jobExecution.getJobParameters()) - .getJobInstance().getInstanceId(); - assertNotEquals(instanceIdSecondJob, instanceIdFirstJob); - MongoTestDatabaseSetup.assertPostDeprecationDatabaseState(this.mongoTemplate); - } } diff --git a/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/configuration/steps/DeprecateStudySubmittedVariantsStepConfigurationTest.java b/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/configuration/steps/DeprecateStudySubmittedVariantsStepConfigurationTest.java index 67ad9f95a..7328fc11a 100644 --- a/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/configuration/steps/DeprecateStudySubmittedVariantsStepConfigurationTest.java +++ b/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/configuration/steps/DeprecateStudySubmittedVariantsStepConfigurationTest.java @@ -28,19 +28,19 @@ import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit.jupiter.SpringExtension; +import uk.ac.ebi.eva.accession.core.configuration.ContiguousIdBlocksDataSourceConfiguration; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.deprecate.MongoTestDatabaseSetup; import uk.ac.ebi.eva.accession.deprecate.configuration.BeanNames; -import uk.ac.ebi.eva.accession.deprecate.test.configuration.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.accession.deprecate.test.configuration.BatchTestConfiguration; -import uk.ac.ebi.eva.accession.deprecate.test.configuration.MongoTestConfiguration; import static org.junit.jupiter.api.Assertions.assertEquals; import static uk.ac.ebi.eva.accession.deprecate.test.configuration.BatchTestConfiguration.JOB_LAUNCHER_FROM_MONGO; @ExtendWith(SpringExtension.class) @ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class, - BatchJobRepositoryTestConfiguration.class}) + ContiguousIdBlocksDataSourceConfiguration.class}) @TestPropertySource("classpath:study-submitted-variants-test.properties") public class DeprecateStudySubmittedVariantsStepConfigurationTest extends MongoTestContainerHelper { @Autowired diff --git a/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/configuration/steps/DeprecateSubmittedVariantsFromFileStepConfigurationTest.java b/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/configuration/steps/DeprecateSubmittedVariantsFromFileStepConfigurationTest.java index 05ab6f356..6bcae4b68 100644 --- a/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/configuration/steps/DeprecateSubmittedVariantsFromFileStepConfigurationTest.java +++ b/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/configuration/steps/DeprecateSubmittedVariantsFromFileStepConfigurationTest.java @@ -28,19 +28,19 @@ import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit.jupiter.SpringExtension; +import uk.ac.ebi.eva.accession.core.configuration.ContiguousIdBlocksDataSourceConfiguration; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.deprecate.MongoTestDatabaseSetup; import uk.ac.ebi.eva.accession.deprecate.configuration.BeanNames; -import uk.ac.ebi.eva.accession.deprecate.test.configuration.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.accession.deprecate.test.configuration.BatchTestConfiguration; -import uk.ac.ebi.eva.accession.deprecate.test.configuration.MongoTestConfiguration; import static org.junit.jupiter.api.Assertions.assertEquals; import static uk.ac.ebi.eva.accession.deprecate.test.configuration.BatchTestConfiguration.JOB_LAUNCHER_FROM_FILE; @ExtendWith(SpringExtension.class) @ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class, - BatchJobRepositoryTestConfiguration.class}) + ContiguousIdBlocksDataSourceConfiguration.class}) @TestPropertySource("classpath:deprecate-submitted-variants-from-file-test.properties") public class DeprecateSubmittedVariantsFromFileStepConfigurationTest extends MongoTestContainerHelper { @Autowired diff --git a/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/test/configuration/BatchJobRepositoryTestConfiguration.java b/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/test/configuration/BatchJobRepositoryTestConfiguration.java deleted file mode 100644 index 854541a72..000000000 --- a/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/test/configuration/BatchJobRepositoryTestConfiguration.java +++ /dev/null @@ -1,22 +0,0 @@ -package uk.ac.ebi.eva.accession.deprecate.test.configuration; - -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Primary; -import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseBuilder; -import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseType; - -import javax.sql.DataSource; - -@Configuration -public class BatchJobRepositoryTestConfiguration { - - @Bean - @Primary - public DataSource dataSource() { - return new EmbeddedDatabaseBuilder() - .setType(EmbeddedDatabaseType.HSQL) - .addScript("org/springframework/batch/core/schema-hsqldb.sql") - .build(); - } -} \ No newline at end of file diff --git a/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/test/configuration/BatchTestConfiguration.java b/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/test/configuration/BatchTestConfiguration.java index f30f6de12..bc56c123d 100644 --- a/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/test/configuration/BatchTestConfiguration.java +++ b/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/test/configuration/BatchTestConfiguration.java @@ -20,9 +20,10 @@ import org.springframework.batch.core.repository.JobRepository; import org.springframework.batch.test.JobLauncherTestUtils; import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Import; +import org.springframework.web.client.RestTemplate; +import uk.ac.ebi.eva.accession.core.configuration.InMemoryBatchConfiguration; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.MongoConfiguration; import uk.ac.ebi.eva.accession.deprecate.configuration.InputParametersConfiguration; import uk.ac.ebi.eva.accession.deprecate.configuration.batch.io.StudySubmittedVariantsFileReaderConfiguration; @@ -38,8 +39,8 @@ import static uk.ac.ebi.eva.accession.deprecate.configuration.BeanNames.DEPRECATE_STUDY_SUBMITTED_VARIANTS_JOB; import static uk.ac.ebi.eva.accession.deprecate.configuration.BeanNames.DEPRECATE_SUBMITTED_VARIANTS_FROM_FILE_JOB; -@EnableAutoConfiguration -@Import({MongoConfiguration.class, +@Import({InMemoryBatchConfiguration.class, + MongoConfiguration.class, InputParametersConfiguration.class, ChunkSizeCompletionPolicyConfiguration.class, StudySubmittedVariantsReaderConfiguration.class, @@ -75,4 +76,11 @@ public JobLauncherTestUtils jobLauncherTestUtilsFromFile(JobLauncher jobLauncher return utils; } + + @Bean(name = "COUNT_STATS_REST_TEMPLATE") + public RestTemplate countStatsRestTemplate() { + return new RestTemplate(); + } + + } diff --git a/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/test/configuration/MongoTestConfiguration.java b/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/test/configuration/MongoTestConfiguration.java deleted file mode 100644 index 08c2a4774..000000000 --- a/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/test/configuration/MongoTestConfiguration.java +++ /dev/null @@ -1,105 +0,0 @@ -/* - * - * Copyright 2022 EMBL - European Bioinformatics Institute - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -package uk.ac.ebi.eva.accession.deprecate.test.configuration; - -import com.mongodb.ReadPreference; -import com.mongodb.WriteConcern; -import com.mongodb.client.MongoClient; -import com.mongodb.client.MongoClients; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.domain.EntityScan; -import org.springframework.boot.test.autoconfigure.data.mongo.AutoConfigureDataMongo; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Primary; -import org.springframework.data.mongodb.MongoDatabaseFactory; -import org.springframework.data.mongodb.config.EnableMongoAuditing; -import org.springframework.data.mongodb.core.MongoTemplate; -import org.springframework.data.mongodb.core.SimpleMongoClientDatabaseFactory; -import org.springframework.data.mongodb.core.WriteResultChecking; -import org.springframework.data.mongodb.core.convert.DefaultDbRefResolver; -import org.springframework.data.mongodb.core.convert.DefaultMongoTypeMapper; -import org.springframework.data.mongodb.core.convert.MappingMongoConverter; -import org.springframework.data.mongodb.core.convert.MongoCustomConversions; -import org.springframework.data.mongodb.core.mapping.MongoMappingContext; -import org.springframework.data.mongodb.repository.config.EnableMongoRepositories; - -import java.util.Collections; - -@Configuration -@EntityScan(basePackages = {"uk.ac.ebi.eva.accession.core.persistence"}) -@EnableMongoRepositories(basePackages = { - "uk.ac.ebi.eva.accession.core.persistence", - "uk.ac.ebi.ampt2d.commons.accession.persistence.mongodb.repository"}) -@EnableMongoAuditing -@AutoConfigureDataMongo -public class MongoTestConfiguration { - - @Value("${spring.data.mongodb.uri}") - private String mongoUri; - - @Value("${spring.data.mongodb.database}") - private String database; - - @Value("${mongodb.read-preference:primary}") - private String readPreference; - - @Bean - @Primary - public MongoClient mongoClient() { - return MongoClients.create(mongoUri); - } - - @Bean("primaryFactory") - @Primary - public MongoDatabaseFactory mongoDbFactory() { - return new SimpleMongoClientDatabaseFactory(mongoClient(), database); - } - - @Bean - @Primary - public MongoCustomConversions mongoCustomConversions() { - return new MongoCustomConversions(Collections.emptyList()); - } - - @Bean - @Primary - public MongoMappingContext mongoMappingContext(MongoCustomConversions conversions) { - MongoMappingContext context = new MongoMappingContext(); - context.setSimpleTypeHolder(conversions.getSimpleTypeHolder()); - context.setAutoIndexCreation(false); - return context; - } - - @Bean - @Primary - public MongoTemplate mongoTemplate(MongoMappingContext mongoMappingContext, - MongoCustomConversions conversions) { - MappingMongoConverter converter = new MappingMongoConverter( - new DefaultDbRefResolver(mongoDbFactory()), mongoMappingContext); - converter.setCustomConversions(conversions); - converter.setTypeMapper(new DefaultMongoTypeMapper(null)); - converter.afterPropertiesSet(); - MongoTemplate mongoTemplate = new MongoTemplate(mongoDbFactory(), converter); - mongoTemplate.setReadPreference(ReadPreference.valueOf(readPreference)); - mongoTemplate.setWriteConcern(WriteConcern.MAJORITY); - mongoTemplate.setWriteResultChecking(WriteResultChecking.EXCEPTION); - return mongoTemplate; - } - -} \ No newline at end of file diff --git a/eva-accession-deprecate/src/test/resources/deprecate-submitted-variants-from-file-test.properties b/eva-accession-deprecate/src/test/resources/deprecate-submitted-variants-from-file-test.properties index 5ea0e7638..097a7e3c1 100644 --- a/eva-accession-deprecate/src/test/resources/deprecate-submitted-variants-from-file-test.properties +++ b/eva-accession-deprecate/src/test/resources/deprecate-submitted-variants-from-file-test.properties @@ -14,6 +14,11 @@ accessioning.instanceId=test-instance-01 accessioning.submitted.categoryId=test-ss accessioning.clustered.categoryId=test-rs +continuous.id.blocks.datasource.driver-class-name=org.hsqldb.jdbc.JDBCDriver +continuous.id.blocks.datasource.jdbc-url=jdbc:hsqldb:mem:db;sql.syntax_pgs=true;DB_CLOSE_DELAY=-1 +continuous.id.blocks.datasource.username=SA +continuous.id.blocks.datasource.password= + accessioning.monotonic.test-ss.blockSize=100000 accessioning.monotonic.test-ss.blockStartValue=4 accessioning.monotonic.test-ss.nextBlockInterval=1000000000 diff --git a/eva-accession-deprecate/src/test/resources/study-submitted-variants-test.properties b/eva-accession-deprecate/src/test/resources/study-submitted-variants-test.properties index e0e437959..c70a51775 100644 --- a/eva-accession-deprecate/src/test/resources/study-submitted-variants-test.properties +++ b/eva-accession-deprecate/src/test/resources/study-submitted-variants-test.properties @@ -11,6 +11,11 @@ accessioning.instanceId=test-instance-01 accessioning.submitted.categoryId=test-ss accessioning.clustered.categoryId=test-rs +continuous.id.blocks.datasource.driver-class-name=org.hsqldb.jdbc.JDBCDriver +continuous.id.blocks.datasource.jdbc-url=jdbc:hsqldb:mem:db;sql.syntax_pgs=true;DB_CLOSE_DELAY=-1 +continuous.id.blocks.datasource.username=SA +continuous.id.blocks.datasource.password= + accessioning.monotonic.test-ss.blockSize=100000 accessioning.monotonic.test-ss.blockStartValue=4 accessioning.monotonic.test-ss.nextBlockInterval=1000000000 diff --git a/eva-accession-import-dbsnp2/pom.xml b/eva-accession-import-dbsnp2/pom.xml index 03889334e..4e0670781 100644 --- a/eva-accession-import-dbsnp2/pom.xml +++ b/eva-accession-import-dbsnp2/pom.xml @@ -44,6 +44,11 @@ spring-batch-test test + + org.hsqldb + hsqldb + test + org.springframework.boot spring-boot-testcontainers @@ -72,11 +77,6 @@ uk.ac.ebi.eva variation-commons-batch - - com.h2database - h2 - runtime - diff --git a/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/Application.java b/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/Application.java index f5e225871..350ea8044 100644 --- a/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/Application.java +++ b/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/Application.java @@ -17,9 +17,23 @@ import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.autoconfigure.batch.BatchAutoConfiguration; +import org.springframework.boot.autoconfigure.data.mongo.MongoDataAutoConfiguration; +import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration; +import org.springframework.boot.autoconfigure.jdbc.DataSourceTransactionManagerAutoConfiguration; +import org.springframework.boot.autoconfigure.orm.jpa.HibernateJpaAutoConfiguration; import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.context.annotation.Import; +import uk.ac.ebi.eva.accession.core.configuration.InMemoryBatchConfiguration; -@SpringBootApplication +@SpringBootApplication(exclude = { + MongoDataAutoConfiguration.class, + DataSourceAutoConfiguration.class, + DataSourceTransactionManagerAutoConfiguration.class, + HibernateJpaAutoConfiguration.class, + BatchAutoConfiguration.class +}) +@Import(InMemoryBatchConfiguration.class) public class Application { public static void main(String[] args) { diff --git a/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/configuration/batch/jobs/ImportDbsnpJsonVariantsJobConfiguration.java b/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/configuration/batch/jobs/ImportDbsnpJsonVariantsJobConfiguration.java index d747cfa86..c28200fd8 100644 --- a/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/configuration/batch/jobs/ImportDbsnpJsonVariantsJobConfiguration.java +++ b/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/configuration/batch/jobs/ImportDbsnpJsonVariantsJobConfiguration.java @@ -16,10 +16,8 @@ package uk.ac.ebi.eva.accession.dbsnp2.configuration.batch.jobs; import org.springframework.batch.core.Job; -import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.batch.core.job.builder.JobBuilder; import org.springframework.batch.core.job.flow.Flow; -import org.springframework.batch.core.launch.support.RunIdIncrementer; import org.springframework.batch.core.repository.JobRepository; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; @@ -33,7 +31,6 @@ * Configuration to run a job that imports a dbSNP JSON file. */ @Configuration -@EnableBatchProcessing public class ImportDbsnpJsonVariantsJobConfiguration { @Autowired @@ -43,7 +40,6 @@ public class ImportDbsnpJsonVariantsJobConfiguration { @Bean(IMPORT_DBSNP_JSON_VARIANTS_JOB) public Job importDbsnpJsonVariantsJob(JobRepository jobRepository) { return new JobBuilder(IMPORT_DBSNP_JSON_VARIANTS_JOB, jobRepository) - .incrementer(new RunIdIncrementer()) .start(importFlow) .end() .build(); diff --git a/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/configuration/batch/steps/ImportDbsnpJsonVariantsStepConfiguration.java b/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/configuration/batch/steps/ImportDbsnpJsonVariantsStepConfiguration.java index 0828b2229..501c10e33 100644 --- a/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/configuration/batch/steps/ImportDbsnpJsonVariantsStepConfiguration.java +++ b/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/configuration/batch/steps/ImportDbsnpJsonVariantsStepConfiguration.java @@ -18,7 +18,6 @@ import com.fasterxml.jackson.databind.JsonNode; import org.springframework.batch.core.Step; import org.springframework.batch.core.StepExecutionListener; -import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.batch.core.repository.JobRepository; import org.springframework.batch.core.step.builder.StepBuilder; import org.springframework.batch.item.ItemProcessor; @@ -32,6 +31,7 @@ import org.springframework.transaction.PlatformTransactionManager; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpClusteredVariantEntity; +import static uk.ac.ebi.eva.accession.core.configuration.InMemoryBatchConfiguration.BATCH_TRANSACTION_MANAGER; import static uk.ac.ebi.eva.accession.dbsnp2.configuration.BeanNames.DBSNP_JSON_VARIANT_PROCESSOR; import static uk.ac.ebi.eva.accession.dbsnp2.configuration.BeanNames.DBSNP_JSON_VARIANT_READER; import static uk.ac.ebi.eva.accession.dbsnp2.configuration.BeanNames.DBSNP_JSON_VARIANT_WRITER; @@ -42,7 +42,6 @@ * Configuration for dbSNP JSON import flow step */ @Configuration -@EnableBatchProcessing public class ImportDbsnpJsonVariantsStepConfiguration { @Autowired @@ -63,7 +62,8 @@ public class ImportDbsnpJsonVariantsStepConfiguration { @Bean(IMPORT_DBSNP_JSON_VARIANTS_STEP) - public Step importDbsnpJsonVariantsStep(JobRepository jobRepository, PlatformTransactionManager transactionManager, + public Step importDbsnpJsonVariantsStep(JobRepository jobRepository, + @Qualifier(BATCH_TRANSACTION_MANAGER) PlatformTransactionManager transactionManager, SimpleCompletionPolicy chunkSizeCompletionPolicy) { return new StepBuilder(IMPORT_DBSNP_JSON_VARIANTS_STEP, jobRepository) .chunk(chunkSizeCompletionPolicy, transactionManager) diff --git a/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/parameters/InputParameters.java b/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/parameters/InputParameters.java index 641367832..53d70ef16 100644 --- a/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/parameters/InputParameters.java +++ b/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/parameters/InputParameters.java @@ -27,16 +27,15 @@ public class InputParameters { private int previousImportedBuild; private boolean incrementalImport; private int chunkSize; - private boolean forceRestart; private boolean forceImport; public JobParameters toJobParameters() { return new JobParametersBuilder() - .addString("input", input) - .addString("genbankAssembly", genbankAssembly) - .addString("refseqAssembly", refseqAssembly) - .addLong("chunkSize", (long) chunkSize, false) - .toJobParameters(); + .addString("input", input) + .addString("genbankAssembly", genbankAssembly) + .addString("refseqAssembly", refseqAssembly) + .addLong("chunkSize", (long) chunkSize, false) + .toJobParameters(); } public String getInput() { @@ -95,14 +94,6 @@ public void setChunkSize(int chunkSize) { this.chunkSize = chunkSize; } - public boolean isForceRestart() { - return forceRestart; - } - - public void setForceRestart(boolean forceRestart) { - this.forceRestart = forceRestart; - } - public boolean isForceImport() { return forceImport; } diff --git a/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/runner/DbsnpJsonImportVariantsJobLauncherCommandLineRunner.java b/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/runner/DbsnpJsonImportVariantsJobLauncherCommandLineRunner.java index ff80ea812..e297d6770 100644 --- a/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/runner/DbsnpJsonImportVariantsJobLauncherCommandLineRunner.java +++ b/eva-accession-import-dbsnp2/src/main/java/uk/ac/ebi/eva/accession/dbsnp2/runner/DbsnpJsonImportVariantsJobLauncherCommandLineRunner.java @@ -18,7 +18,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.batch.core.Job; -import org.springframework.batch.core.JobExecution; import org.springframework.batch.core.JobExecutionException; import org.springframework.batch.core.JobParameters; import org.springframework.batch.core.JobParametersInvalidException; @@ -35,11 +34,9 @@ import org.springframework.context.ApplicationEventPublisherAware; import org.springframework.stereotype.Component; import org.springframework.util.PatternMatchUtils; -import uk.ac.ebi.eva.accession.core.runner.CommandLineRunnerUtils; import uk.ac.ebi.eva.accession.dbsnp2.parameters.InputParameters; import uk.ac.ebi.eva.commons.batch.exception.NoJobToExecuteException; import uk.ac.ebi.eva.commons.batch.exception.NoParametersHaveBeenPassedException; -import uk.ac.ebi.eva.commons.batch.exception.NoPreviousJobExecutionException; import uk.ac.ebi.eva.commons.batch.job.JobExecutionApplicationListener; import uk.ac.ebi.eva.commons.batch.job.JobStatusManager; @@ -115,17 +112,10 @@ public void run(String... args) throws JobExecutionException { JobStatusManager.checkIfJobNameHasBeenDefined(jobName); JobStatusManager.checkIfPropertiesHaveBeenProvided(jobParameters); - if (inputParameters.isForceRestart()) { - JobExecution previousJobExecution = CommandLineRunnerUtils.getLastJobExecution(jobName, jobExplorer, - jobParameters); - CommandLineRunnerUtils.markPreviousJobAsFailed(jobName, jobRepository, - previousJobExecution != null ? previousJobExecution.getJobParameters() : jobParameters); - } else { - jobParameters = CommandLineRunnerUtils.addRunIDToJobParameters(jobName, jobExplorer, jobParameters); - } + launchJob(jobParameters); - } catch (NoPreviousJobExecutionException | NoParametersHaveBeenPassedException | NoJobToExecuteException - | JobParametersInvalidException | JobExecutionAlreadyRunningException e) { + } catch (NoParametersHaveBeenPassedException | NoJobToExecuteException | JobParametersInvalidException + | JobExecutionAlreadyRunningException e) { logger.error(e.getMessage()); logger.debug("Error trace", e); abnormalExit = true; diff --git a/eva-accession-import-dbsnp2/src/main/resources/application.properties b/eva-accession-import-dbsnp2/src/main/resources/application.properties index 2730fabe3..f4b6ab7dc 100644 --- a/eva-accession-import-dbsnp2/src/main/resources/application.properties +++ b/eva-accession-import-dbsnp2/src/main/resources/application.properties @@ -10,7 +10,6 @@ parameters.assemblyReportUrl= parameters.previousImportedBuild= parameters.incrementalImport=false parameters.chunkSize= -parameters.forceRestart=false # MongoDB for storing imported accessions spring.data.mongodb.host= diff --git a/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/batch/io/DbsnpJsonClusteredVariantsWriterTest.java b/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/batch/io/DbsnpJsonClusteredVariantsWriterTest.java index 8c2b413dd..686a52204 100644 --- a/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/batch/io/DbsnpJsonClusteredVariantsWriterTest.java +++ b/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/batch/io/DbsnpJsonClusteredVariantsWriterTest.java @@ -41,9 +41,9 @@ import uk.ac.ebi.eva.accession.core.repository.nonhuman.dbsnp.DbsnpClusteredVariantAccessioningRepository; import uk.ac.ebi.eva.accession.core.repository.nonhuman.dbsnp.DbsnpClusteredVariantOperationRepository; import uk.ac.ebi.eva.accession.core.summary.ClusteredVariantSummaryFunction; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.dbsnp2.test.BatchTestConfiguration; -import uk.ac.ebi.eva.accession.dbsnp2.test.MongoTestConfiguration; import uk.ac.ebi.eva.commons.core.models.VariantType; import java.time.LocalDateTime; diff --git a/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/configuration/jobs/ImportDbsnpJsonVariantsJobConfigurationTest.java b/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/configuration/jobs/ImportDbsnpJsonVariantsJobConfigurationTest.java index eb4fcded3..f1d3a1792 100644 --- a/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/configuration/jobs/ImportDbsnpJsonVariantsJobConfigurationTest.java +++ b/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/configuration/jobs/ImportDbsnpJsonVariantsJobConfigurationTest.java @@ -27,11 +27,10 @@ import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit.jupiter.SpringExtension; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.dbsnp2.parameters.InputParameters; -import uk.ac.ebi.eva.accession.dbsnp2.test.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.accession.dbsnp2.test.BatchTestConfiguration; -import uk.ac.ebi.eva.accession.dbsnp2.test.MongoTestConfiguration; import java.util.Collection; import java.util.Collections; @@ -43,8 +42,7 @@ import static uk.ac.ebi.eva.accession.dbsnp2.test.BatchTestConfiguration.JOB_IMPORT_DBSNP_JSON_VARIANTS_JOB; @ExtendWith(SpringExtension.class) -@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class, - BatchJobRepositoryTestConfiguration.class}) +@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class}) @TestPropertySource("classpath:application.properties") public class ImportDbsnpJsonVariantsJobConfigurationTest extends MongoTestContainerHelper { diff --git a/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/configuration/steps/ImportDbsnpJsonVariantsStepConfigurationTest.java b/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/configuration/steps/ImportDbsnpJsonVariantsStepConfigurationTest.java index 06522e338..666df1f93 100644 --- a/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/configuration/steps/ImportDbsnpJsonVariantsStepConfigurationTest.java +++ b/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/configuration/steps/ImportDbsnpJsonVariantsStepConfigurationTest.java @@ -30,18 +30,16 @@ import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpClusteredVariantEntity; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; -import uk.ac.ebi.eva.accession.dbsnp2.test.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.accession.dbsnp2.test.BatchTestConfiguration; -import uk.ac.ebi.eva.accession.dbsnp2.test.MongoTestConfiguration; import static org.junit.jupiter.api.Assertions.assertEquals; import static uk.ac.ebi.eva.accession.dbsnp2.configuration.BeanNames.IMPORT_DBSNP_JSON_VARIANTS_STEP; import static uk.ac.ebi.eva.accession.dbsnp2.test.BatchTestConfiguration.JOB_IMPORT_DBSNP_JSON_VARIANTS_JOB; @ExtendWith(SpringExtension.class) -@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class, - BatchJobRepositoryTestConfiguration.class}) +@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class}) @TestPropertySource("classpath:application.properties") public class ImportDbsnpJsonVariantsStepConfigurationTest extends MongoTestContainerHelper { diff --git a/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/runner/DbsnpJsonImportVariantsJobLauncherCommandLineRunnerTest.java b/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/runner/DbsnpJsonImportVariantsJobLauncherCommandLineRunnerTest.java index 3add32b31..611ee062c 100644 --- a/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/runner/DbsnpJsonImportVariantsJobLauncherCommandLineRunnerTest.java +++ b/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/runner/DbsnpJsonImportVariantsJobLauncherCommandLineRunnerTest.java @@ -22,8 +22,6 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; -import org.springframework.batch.core.JobInstance; -import org.springframework.batch.core.StepExecution; import org.springframework.batch.core.explore.JobExplorer; import org.springframework.batch.core.repository.JobRepository; import org.springframework.batch.item.file.FlatFileItemReader; @@ -34,13 +32,11 @@ import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit.jupiter.SpringExtension; -import uk.ac.ebi.eva.accession.core.runner.CommandLineRunnerUtils; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.dbsnp2.batch.io.BzipLazyResource; import uk.ac.ebi.eva.accession.dbsnp2.parameters.InputParameters; -import uk.ac.ebi.eva.accession.dbsnp2.test.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.accession.dbsnp2.test.BatchTestConfiguration; -import uk.ac.ebi.eva.accession.dbsnp2.test.MongoTestConfiguration; import uk.ac.ebi.eva.commons.core.utils.FileUtils; import java.io.BufferedReader; @@ -50,19 +46,14 @@ import java.io.InputStreamReader; import java.io.OutputStream; import java.nio.charset.StandardCharsets; -import java.util.Arrays; import java.util.Collections; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotEquals; import static uk.ac.ebi.eva.accession.dbsnp2.configuration.BeanNames.IMPORT_DBSNP_JSON_VARIANTS_JOB; -import static uk.ac.ebi.eva.accession.dbsnp2.configuration.BeanNames.IMPORT_DBSNP_JSON_VARIANTS_STEP; import static uk.ac.ebi.eva.accession.dbsnp2.runner.DbsnpJsonImportVariantsJobLauncherCommandLineRunner.EXIT_WITHOUT_ERRORS; -import static uk.ac.ebi.eva.accession.dbsnp2.runner.DbsnpJsonImportVariantsJobLauncherCommandLineRunner.EXIT_WITH_ERRORS; @ExtendWith(SpringExtension.class) -@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class, - BatchJobRepositoryTestConfiguration.class}) +@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class}) @TestPropertySource("classpath:application.properties") @SpringBatchTest public class DbsnpJsonImportVariantsJobLauncherCommandLineRunnerTest extends MongoTestContainerHelper { @@ -113,14 +104,12 @@ public void setUp() throws Exception { jobRepositoryTestUtils = new JobRepositoryTestUtils(jobRepository); runner.setJobName(IMPORT_DBSNP_JSON_VARIANTS_JOB); jobRepositoryTestUtils.removeJobExecutions(); - inputParameters.setForceRestart(false); useOriginalJsonFile(); } @AfterEach public void tearDown() { jobRepositoryTestUtils.removeJobExecutions(); - inputParameters.setForceRestart(false); } @Test @@ -137,142 +126,24 @@ public void restartCompletedJobThatIsAlreadyInTheRepository() throws Exception { runner.run(); assertEquals(EXIT_WITHOUT_ERRORS, runner.getExitCode()); - inputParameters.setForceRestart(true); runner.run(); assertEquals(EXIT_WITHOUT_ERRORS, runner.getExitCode()); } - @Test - @DirtiesContext - public void restartFailedJobThatIsAlreadyInTheRepository() throws Exception { - useTempJsonFile(); - injectErrorIntoTempJson(); - JobInstance failingJobInstance = runJobAandCheckResults(); - - inputParameters.setForceRestart(true); - remediateTempVcfError(); - runJobBAndCheckRestart(failingJobInstance); - } - - private JobInstance runJobAandCheckResults() throws Exception { - runner.run(); - assertEquals(EXIT_WITH_ERRORS, runner.getExitCode()); - JobInstance currentJobInstance = CommandLineRunnerUtils.getLastJobExecution(IMPORT_DBSNP_JSON_VARIANTS_JOB, - jobExplorer, - inputParameters.toJobParameters()) - .getJobInstance(); - StepExecution stepExecution = jobRepository.getLastStepExecution(currentJobInstance, - IMPORT_DBSNP_JSON_VARIANTS_STEP); - //Ensure that only the first batch was written (batch size is 2 and error was at line#4) - assertEquals(inputParameters.getChunkSize(), stepExecution.getWriteCount()); - - return currentJobInstance; - } - - private void runJobBAndCheckRestart(JobInstance failingJobInstance) throws Exception { - runner.run(); - assertEquals(EXIT_WITHOUT_ERRORS, runner.getExitCode()); - JobInstance currentJobInstance = CommandLineRunnerUtils.getLastJobExecution(IMPORT_DBSNP_JSON_VARIANTS_JOB, - jobExplorer, - inputParameters.toJobParameters()) - .getJobInstance(); - assertNotEquals(failingJobInstance.getInstanceId(), currentJobInstance.getInstanceId()); - } - @Test @DirtiesContext public void forceRestartButNoJobInTheRepository() throws Exception { - inputParameters.setForceRestart(true); assertEquals(Collections.EMPTY_LIST, jobExplorer.getJobNames()); runner.run(); assertEquals(EXIT_WITHOUT_ERRORS, runner.getExitCode()); } - @Test - @DirtiesContext - public void resumeFailingJobFromCorrectChunk() throws Exception { - // Jobs A, B, C are run chronological order; A and C have SAME parameters; - // A is the job that is run after VCF fault injection (as part of the runTestWithFaultInjection method), - // therefore should fail. - // B is a job run with the original VCF without any faults (run separately), therefore should succeed. - // C is a job with the same parameters as A run after VCF fault remediation (as part of the - // runTestWithFaultInjection method), therefore should resume A and succeed. - - useTempJsonFile(); - injectErrorIntoTempJson(); - JobInstance failingJobInstance = runJobAandCheckResults(); - - runJobBAndCheckResults(); - - remediateTempVcfError(); - runJobCAndCheckResumption(failingJobInstance); - } - - private void runJobBAndCheckResults() throws Exception { - useOriginalJsonFile(); - runner.run(); - assertEquals(EXIT_WITHOUT_ERRORS, runner.getExitCode()); - - //Restore state so that Job C can continue running after fault remediation - useTempJsonFile(); - } - - private void runJobCAndCheckResumption(JobInstance failingJobInstance) throws Exception { - runner.run(); - JobInstance currentJobInstance = CommandLineRunnerUtils.getLastJobExecution(IMPORT_DBSNP_JSON_VARIANTS_JOB, - jobExplorer, - inputParameters.toJobParameters()) - .getJobInstance(); - StepExecution stepExecution = jobRepository.getLastStepExecution(currentJobInstance, - IMPORT_DBSNP_JSON_VARIANTS_STEP); - // Did we resume the previous failed job instance? - assertEquals(failingJobInstance.getInstanceId(), currentJobInstance.getInstanceId()); - - int numberOfLinesInJson = getNumberOfLinesInJsonString(originalJsonContent); - // Test resumption point - did we pick up where we left off? - // Ensure all the batches other than the first batch were processed - assertEquals(numberOfLinesInJson - inputParameters.getChunkSize(), stepExecution.getWriteCount()); - assertEquals(EXIT_WITHOUT_ERRORS, runner.getExitCode()); - } - - private void injectErrorIntoTempJson() throws Exception { - // Intentionally inject error in entry#9 in the original Json - String modifiedJsonContent = originalJsonContent.replace("3069077", "3069077jibberish"); - writeToTempJsonFile(modifiedJsonContent); - } - - private void remediateTempVcfError() throws Exception { - writeToTempJsonFile(originalJsonContent); - } - private void useOriginalJsonFile() throws Exception { inputParameters.setInput(originalJsonInputFilePath); variantReader.setResource(FileUtils.getResource(new File(originalJsonInputFilePath))); } - private void useTempJsonFile() throws Exception { - // The following does not actually change the wiring of the variantReader since the wiring happens before the tests - // This setVcf is only to facilitate identifying jobs in the job repo by parameter - // (those that use original vs temp JSON) - inputParameters.setInput(tempJsonInputFileToTestFailingJobs.getAbsolutePath()); - /* - * Change the auto-wired JSON for variantReader at runtime - * Rationale: - * 1) Why not use two test configurations, one for a JSON that fails validation and another for a JSON - * that won't and test resumption? - * Beginning Spring Boot 2, job resumption can only happen when input parameters to the restarted job - * is the same as the failed job. - * Therefore, a test to check resumption cannot have two different config files with different - * parameters.input. - * This test therefore creates a dynamic JSON and injects errors at runtime to the JSON thus preserving - * the input parameter but changing the JSON content. - * 2) Why not artificially inject a variantReader exception? - * This will preclude us from verifying job resumption from a precise line in the JSON. - */ - variantReader.setResource(FileUtils.getResource(tempJsonInputFileToTestFailingJobs)); - } - private void writeToTempJsonFile(String modifiedJsonContent) throws IOException { OutputStream bzipOutputStream = new BzipLazyResource(tempJsonInputFileToTestFailingJobs.getAbsolutePath()).getOutputStream(); @@ -290,10 +161,4 @@ private String getOriginalJsonContent(String inputJsonPath) throws IOException { } return originalJsonContent.toString(); } - - private int getNumberOfLinesInJsonString(String jsonString) { - return (int) Arrays.stream(jsonString.split(System.lineSeparator())) - .filter(line -> !line.startsWith("#")) - .count(); - } } \ No newline at end of file diff --git a/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/test/BatchJobRepositoryTestConfiguration.java b/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/test/BatchJobRepositoryTestConfiguration.java deleted file mode 100644 index dd4a09fb2..000000000 --- a/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/test/BatchJobRepositoryTestConfiguration.java +++ /dev/null @@ -1,22 +0,0 @@ -package uk.ac.ebi.eva.accession.dbsnp2.test; - -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Primary; -import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseBuilder; -import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseType; - -import javax.sql.DataSource; - -@Configuration -public class BatchJobRepositoryTestConfiguration { - - @Bean - @Primary - public DataSource dataSource() { - return new EmbeddedDatabaseBuilder() - .setType(EmbeddedDatabaseType.HSQL) - .addScript("org/springframework/batch/core/schema-hsqldb.sql") - .build(); - } -} \ No newline at end of file diff --git a/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/test/BatchTestConfiguration.java b/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/test/BatchTestConfiguration.java index e88926287..2bfa8dc56 100644 --- a/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/test/BatchTestConfiguration.java +++ b/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/test/BatchTestConfiguration.java @@ -16,15 +16,14 @@ package uk.ac.ebi.eva.accession.dbsnp2.test; import org.springframework.batch.core.Job; -import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.batch.core.launch.JobLauncher; import org.springframework.batch.core.repository.JobRepository; import org.springframework.batch.test.JobLauncherTestUtils; import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Import; +import uk.ac.ebi.eva.accession.core.configuration.InMemoryBatchConfiguration; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.MongoConfiguration; import uk.ac.ebi.eva.accession.dbsnp2.configuration.InputParametersConfiguration; import uk.ac.ebi.eva.accession.dbsnp2.configuration.batch.flow.ImportDbsnpJsonFlowConfiguration; @@ -41,9 +40,8 @@ import static uk.ac.ebi.eva.accession.dbsnp2.configuration.BeanNames.IMPORT_DBSNP_JSON_VARIANTS_JOB; @Configuration -@EnableAutoConfiguration -@EnableBatchProcessing -@Import({ListenersConfiguration.class, +@Import({InMemoryBatchConfiguration.class, + ListenersConfiguration.class, MongoConfiguration.class, ImportDbsnpJsonVariantsJobConfiguration.class, ImportDbsnpJsonVariantsStepConfiguration.class, diff --git a/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/test/MongoTestConfiguration.java b/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/test/MongoTestConfiguration.java deleted file mode 100644 index 63280b6dd..000000000 --- a/eva-accession-import-dbsnp2/src/test/java/uk/ac/ebi/eva/accession/dbsnp2/test/MongoTestConfiguration.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Copyright 2021 EMBL - European Bioinformatics Institute - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package uk.ac.ebi.eva.accession.dbsnp2.test; - -import com.mongodb.ReadPreference; -import com.mongodb.WriteConcern; -import com.mongodb.client.MongoClient; -import com.mongodb.client.MongoClients; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.domain.EntityScan; -import org.springframework.boot.test.autoconfigure.data.mongo.AutoConfigureDataMongo; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Primary; -import org.springframework.data.mongodb.MongoDatabaseFactory; -import org.springframework.data.mongodb.config.EnableMongoAuditing; -import org.springframework.data.mongodb.core.MongoTemplate; -import org.springframework.data.mongodb.core.SimpleMongoClientDatabaseFactory; -import org.springframework.data.mongodb.core.WriteResultChecking; -import org.springframework.data.mongodb.core.convert.DefaultDbRefResolver; -import org.springframework.data.mongodb.core.convert.DefaultMongoTypeMapper; -import org.springframework.data.mongodb.core.convert.MappingMongoConverter; -import org.springframework.data.mongodb.core.convert.MongoCustomConversions; -import org.springframework.data.mongodb.core.mapping.MongoMappingContext; -import org.springframework.data.mongodb.repository.config.EnableMongoRepositories; - -import java.util.Collections; - -@Configuration -@EntityScan(basePackages = {"uk.ac.ebi.eva.accession.core.persistence"}) -@EnableMongoRepositories(basePackages = { - "uk.ac.ebi.eva.accession.core.persistence", - "uk.ac.ebi.ampt2d.commons.accession.persistence.mongodb.repository"}) -@EnableMongoAuditing -@AutoConfigureDataMongo -public class MongoTestConfiguration { - - @Value("${spring.data.mongodb.uri}") - private String mongoUri; - - @Value("${spring.data.mongodb.database}") - private String database; - - @Value("${mongodb.read-preference:primary}") - private String readPreference; - - @Bean - @Primary - public MongoClient mongoClient() { - return MongoClients.create(mongoUri); - } - - @Bean("primaryFactory") - @Primary - public MongoDatabaseFactory mongoDbFactory() { - return new SimpleMongoClientDatabaseFactory(mongoClient(), database); - } - - @Bean - @Primary - public MongoCustomConversions mongoCustomConversions() { - return new MongoCustomConversions(Collections.emptyList()); - } - - @Bean - @Primary - public MongoMappingContext mongoMappingContext(MongoCustomConversions conversions) { - MongoMappingContext context = new MongoMappingContext(); - context.setSimpleTypeHolder(conversions.getSimpleTypeHolder()); - context.setAutoIndexCreation(false); - return context; - } - - @Bean - @Primary - public MongoTemplate mongoTemplate(MongoMappingContext mongoMappingContext, - MongoCustomConversions conversions) { - MappingMongoConverter converter = new MappingMongoConverter( - new DefaultDbRefResolver(mongoDbFactory()), mongoMappingContext); - converter.setCustomConversions(conversions); - converter.setTypeMapper(new DefaultMongoTypeMapper(null)); - converter.afterPropertiesSet(); - MongoTemplate mongoTemplate = new MongoTemplate(mongoDbFactory(), converter); - mongoTemplate.setReadPreference(ReadPreference.valueOf(readPreference)); - mongoTemplate.setWriteConcern(WriteConcern.MAJORITY); - mongoTemplate.setWriteResultChecking(WriteResultChecking.EXCEPTION); - return mongoTemplate; - } -} diff --git a/eva-accession-import-dbsnp2/src/test/resources/application.properties b/eva-accession-import-dbsnp2/src/test/resources/application.properties index d2e66060f..c4197edc6 100644 --- a/eva-accession-import-dbsnp2/src/test/resources/application.properties +++ b/eva-accession-import-dbsnp2/src/test/resources/application.properties @@ -10,7 +10,6 @@ parameters.assemblyReportUrl=file:src/test/resources/input-files/GCF_000001405.3 parameters.previousImportedBuild=10 parameters.incrementalImport=true parameters.chunkSize=5 -parameters.forceRestart=false # MongoDB for storing imported accessions spring.data.mongodb.database=admin diff --git a/eva-accession-pipeline/pom.xml b/eva-accession-pipeline/pom.xml index 34e3f8e38..750a7edd9 100644 --- a/eva-accession-pipeline/pom.xml +++ b/eva-accession-pipeline/pom.xml @@ -45,6 +45,11 @@ test + + org.hsqldb + hsqldb + test + org.springframework.batch spring-batch-test diff --git a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/Application.java b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/Application.java index c707ac126..ccdafb238 100644 --- a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/Application.java +++ b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/Application.java @@ -15,14 +15,26 @@ */ package uk.ac.ebi.eva.accession.pipeline; -import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.autoconfigure.batch.BatchAutoConfiguration; +import org.springframework.boot.autoconfigure.data.mongo.MongoDataAutoConfiguration; +import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration; +import org.springframework.boot.autoconfigure.jdbc.DataSourceTransactionManagerAutoConfiguration; +import org.springframework.boot.autoconfigure.orm.jpa.HibernateJpaAutoConfiguration; import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.context.annotation.Import; +import uk.ac.ebi.eva.accession.core.configuration.ContiguousIdBlocksDataSourceConfiguration; +import uk.ac.ebi.eva.accession.core.configuration.InMemoryBatchConfiguration; -// TODO: eva-pipeline is also excluding MongoDataAutoConfiguration -@EnableBatchProcessing -@SpringBootApplication +@SpringBootApplication(exclude = { + MongoDataAutoConfiguration.class, + DataSourceAutoConfiguration.class, + DataSourceTransactionManagerAutoConfiguration.class, + HibernateJpaAutoConfiguration.class, + BatchAutoConfiguration.class +}) +@Import({InMemoryBatchConfiguration.class, ContiguousIdBlocksDataSourceConfiguration.class}) public class Application { public static void main(String[] args) { diff --git a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/DuplicateSSAccQCJobConfiguration.java b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/DuplicateSSAccQCJobConfiguration.java index b3f570c39..185f15e4d 100644 --- a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/DuplicateSSAccQCJobConfiguration.java +++ b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/DuplicateSSAccQCJobConfiguration.java @@ -2,9 +2,7 @@ import org.springframework.batch.core.Job; import org.springframework.batch.core.Step; -import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.batch.core.job.builder.JobBuilder; -import org.springframework.batch.core.launch.support.RunIdIncrementer; import org.springframework.batch.core.repository.JobRepository; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; @@ -15,7 +13,6 @@ import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.DUPLICATE_SS_ACC_QC_STEP; @Configuration -@EnableBatchProcessing public class DuplicateSSAccQCJobConfiguration { @Autowired @@ -25,7 +22,6 @@ public class DuplicateSSAccQCJobConfiguration { @Bean(DUPLICATE_SS_ACC_QC_JOB) public Job duplicateSSAccQCJob(JobRepository jobRepository) { return new JobBuilder(DUPLICATE_SS_ACC_QC_JOB, jobRepository) - .incrementer(new RunIdIncrementer()) .start(duplicateSSAccQCStep) .build(); } diff --git a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/QCSubsnpAccessionsJobConfiguration.java b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/QCSubsnpAccessionsJobConfiguration.java index 6ee75487b..d85368080 100644 --- a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/QCSubsnpAccessionsJobConfiguration.java +++ b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/QCSubsnpAccessionsJobConfiguration.java @@ -2,9 +2,7 @@ import org.springframework.batch.core.Job; import org.springframework.batch.core.Step; -import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.batch.core.job.builder.JobBuilder; -import org.springframework.batch.core.launch.support.RunIdIncrementer; import org.springframework.batch.core.repository.JobRepository; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; @@ -15,7 +13,6 @@ import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.QC_SUBSNP_ACCESSION_STEP; @Configuration -@EnableBatchProcessing public class QCSubsnpAccessionsJobConfiguration { @Autowired @Qualifier(QC_SUBSNP_ACCESSION_STEP) @@ -24,7 +21,6 @@ public class QCSubsnpAccessionsJobConfiguration { @Bean(QC_SUBSNP_ACCESSION_JOB) public Job qcSubsnpAccessionJob(JobRepository jobRepository) { return new JobBuilder(QC_SUBSNP_ACCESSION_JOB, jobRepository) - .incrementer(new RunIdIncrementer()) .start(qcSubsnpAccessionStep) .build(); } diff --git a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/SSAccessionRecoveryJobConfiguration.java b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/SSAccessionRecoveryJobConfiguration.java index 878ed8ad9..5c008d250 100644 --- a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/SSAccessionRecoveryJobConfiguration.java +++ b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/SSAccessionRecoveryJobConfiguration.java @@ -3,9 +3,7 @@ import org.springframework.batch.core.Job; import org.springframework.batch.core.JobExecutionListener; import org.springframework.batch.core.Step; -import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.batch.core.job.builder.JobBuilder; -import org.springframework.batch.core.launch.support.RunIdIncrementer; import org.springframework.batch.core.repository.JobRepository; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; @@ -17,7 +15,6 @@ import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.SS_ACCESSION_RECOVERY_STEP; @Configuration -@EnableBatchProcessing public class SSAccessionRecoveryJobConfiguration { @Autowired @@ -31,7 +28,6 @@ public class SSAccessionRecoveryJobConfiguration { @Bean(SS_ACCESSION_RECOVERY_JOB) public Job createSSAccessionRecoveryJob(JobRepository jobRepository) { return new JobBuilder(SS_ACCESSION_RECOVERY_JOB, jobRepository) - .incrementer(new RunIdIncrementer()) .start(ssAccessionRecoveryStep) .listener(ssAccessionRecoveryJobListener) .build(); diff --git a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/SubsnpAccessionsJobConfiguration.java b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/SubsnpAccessionsJobConfiguration.java index 43290594d..c65602ce5 100644 --- a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/SubsnpAccessionsJobConfiguration.java +++ b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/SubsnpAccessionsJobConfiguration.java @@ -19,9 +19,7 @@ import org.springframework.batch.core.Job; import org.springframework.batch.core.JobExecutionListener; import org.springframework.batch.core.Step; -import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.batch.core.job.builder.JobBuilder; -import org.springframework.batch.core.launch.support.RunIdIncrementer; import org.springframework.batch.core.repository.JobRepository; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; @@ -35,7 +33,6 @@ import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.SUBSNP_ACCESSION_STEP; @Configuration -@EnableBatchProcessing public class SubsnpAccessionsJobConfiguration { @Autowired @@ -57,7 +54,6 @@ public class SubsnpAccessionsJobConfiguration { @Bean(SUBSNP_ACCESSION_JOB) public Job subsnpAccessionJob(JobRepository jobRepository) { return new JobBuilder(SUBSNP_ACCESSION_JOB, jobRepository) - .incrementer(new RunIdIncrementer()) .start(subsnpAccessionStep) .next(accessioningShutdownStep) .next(buildReportStep) diff --git a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/AccessioningShutdownStepConfiguration.java b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/AccessioningShutdownStepConfiguration.java index ccad6721b..55be0d87e 100644 --- a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/AccessioningShutdownStepConfiguration.java +++ b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/AccessioningShutdownStepConfiguration.java @@ -1,25 +1,26 @@ package uk.ac.ebi.eva.accession.pipeline.configuration.batch.steps; import org.springframework.batch.core.Step; -import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.batch.core.repository.JobRepository; import org.springframework.batch.core.step.builder.StepBuilder; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.transaction.PlatformTransactionManager; import uk.ac.ebi.eva.accession.core.service.nonhuman.SubmittedVariantAccessioningService; +import static uk.ac.ebi.eva.accession.core.configuration.InMemoryBatchConfiguration.BATCH_TRANSACTION_MANAGER; import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.ACCESSIONING_SHUTDOWN_STEP; @Configuration -@EnableBatchProcessing public class AccessioningShutdownStepConfiguration { @Autowired private SubmittedVariantAccessioningService submittedVariantAccessioningService; @Bean(ACCESSIONING_SHUTDOWN_STEP) - public Step accessioningShutDownStep(JobRepository jobRepository, PlatformTransactionManager transactionManager) { + public Step accessioningShutDownStep(JobRepository jobRepository, + @Qualifier(BATCH_TRANSACTION_MANAGER) PlatformTransactionManager transactionManager) { return new StepBuilder(ACCESSIONING_SHUTDOWN_STEP, jobRepository) .tasklet((contribution, chunkContext) -> { submittedVariantAccessioningService.shutDownAccessionGenerator(); diff --git a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/BuildReportStepConfiguration.java b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/BuildReportStepConfiguration.java index e7fe6d462..0855df0bc 100644 --- a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/BuildReportStepConfiguration.java +++ b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/BuildReportStepConfiguration.java @@ -17,11 +17,11 @@ package uk.ac.ebi.eva.accession.pipeline.configuration.batch.steps; import org.springframework.batch.core.Step; -import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.batch.core.repository.JobRepository; import org.springframework.batch.core.step.builder.StepBuilder; import org.springframework.batch.core.step.tasklet.TaskletStep; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.transaction.PlatformTransactionManager; @@ -30,17 +30,18 @@ import java.io.File; +import static uk.ac.ebi.eva.accession.core.configuration.InMemoryBatchConfiguration.BATCH_TRANSACTION_MANAGER; import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.BUILD_REPORT_STEP; @Configuration -@EnableBatchProcessing public class BuildReportStepConfiguration { @Autowired private InputParameters inputParameters; @Bean(BUILD_REPORT_STEP) - public Step buildReportStep(JobRepository jobRepository, PlatformTransactionManager transactionManager) { + public Step buildReportStep(JobRepository jobRepository, + @Qualifier(BATCH_TRANSACTION_MANAGER) PlatformTransactionManager transactionManager) { BuildReportTasklet tasklet = new BuildReportTasklet(new File(inputParameters.getOutputVcf())); TaskletStep step = new StepBuilder(BUILD_REPORT_STEP, jobRepository) .tasklet(tasklet, transactionManager) diff --git a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/DuplicateSSAccQCStepConfiguration.java b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/DuplicateSSAccQCStepConfiguration.java index 75a9cabde..26e596671 100644 --- a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/DuplicateSSAccQCStepConfiguration.java +++ b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/DuplicateSSAccQCStepConfiguration.java @@ -1,7 +1,6 @@ package uk.ac.ebi.eva.accession.pipeline.configuration.batch.steps; import org.springframework.batch.core.Step; -import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.batch.core.repository.JobRepository; import org.springframework.batch.core.step.builder.StepBuilder; import org.springframework.batch.core.step.tasklet.TaskletStep; @@ -17,13 +16,13 @@ import java.util.List; +import static uk.ac.ebi.eva.accession.core.configuration.InMemoryBatchConfiguration.BATCH_TRANSACTION_MANAGER; import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.DUPLICATE_SS_ACC_QC_PROCESSOR; import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.DUPLICATE_SS_ACC_QC_STEP; import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.DUPLICATE_SS_ACC_QC_WRITER; import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.SS_ACC_FILE_READER; @Configuration -@EnableBatchProcessing public class DuplicateSSAccQCStepConfiguration { @Autowired @@ -39,7 +38,8 @@ public class DuplicateSSAccQCStepConfiguration { private ItemWriter> duplicateSSAccQCWriter; @Bean(DUPLICATE_SS_ACC_QC_STEP) - public Step duplicateSSAccQCStep(JobRepository jobRepository, PlatformTransactionManager transactionManager) { + public Step duplicateSSAccQCStep(JobRepository jobRepository, + @Qualifier(BATCH_TRANSACTION_MANAGER) PlatformTransactionManager transactionManager) { TaskletStep step = new StepBuilder(DUPLICATE_SS_ACC_QC_STEP, jobRepository) // hardcoded the chunk size as 1, as the reader takes care of accumulating // and sending the chunk size (defined in properties file) elements to the processor diff --git a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/QCSubsnpAccessionsStepConfiguration.java b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/QCSubsnpAccessionsStepConfiguration.java index 6fe42f886..158fc2e83 100644 --- a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/QCSubsnpAccessionsStepConfiguration.java +++ b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/QCSubsnpAccessionsStepConfiguration.java @@ -17,7 +17,6 @@ package uk.ac.ebi.eva.accession.pipeline.configuration.batch.steps; import org.springframework.batch.core.Step; -import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.batch.core.repository.JobRepository; import org.springframework.batch.core.step.builder.StepBuilder; import org.springframework.batch.core.step.tasklet.TaskletStep; @@ -38,12 +37,12 @@ import java.io.File; import java.io.IOException; +import static uk.ac.ebi.eva.accession.core.configuration.InMemoryBatchConfiguration.BATCH_TRANSACTION_MANAGER; import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.QC_SUBSNP_ACCESSION_STEP; import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.REPORT_READER; import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.VARIANT_READER; @Configuration -@EnableBatchProcessing public class QCSubsnpAccessionsStepConfiguration { @Autowired @@ -64,7 +63,8 @@ public ItemStreamReader reportReader() throws IOException { } @Bean(QC_SUBSNP_ACCESSION_STEP) - public Step qcSubsnpAccessionStep(JobRepository jobRepository, PlatformTransactionManager transactionManager) throws IOException { + public Step qcSubsnpAccessionStep(JobRepository jobRepository, + @Qualifier(BATCH_TRANSACTION_MANAGER) PlatformTransactionManager transactionManager) throws IOException { ReportCheckTasklet tasklet = new ReportCheckTasklet(inputReader, reportReader(), inputParameters.getChunkSize() * 2, contigMapping); TaskletStep step = new StepBuilder(QC_SUBSNP_ACCESSION_STEP, jobRepository) diff --git a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/SSAccessionRecoveryStepConfiguration.java b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/SSAccessionRecoveryStepConfiguration.java index a87432c36..a17ae3d03 100644 --- a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/SSAccessionRecoveryStepConfiguration.java +++ b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/SSAccessionRecoveryStepConfiguration.java @@ -1,7 +1,6 @@ package uk.ac.ebi.eva.accession.pipeline.configuration.batch.steps; import org.springframework.batch.core.Step; -import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.batch.core.repository.JobRepository; import org.springframework.batch.core.step.builder.StepBuilder; import org.springframework.beans.factory.annotation.Autowired; @@ -15,7 +14,6 @@ import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.SS_ACCESSION_RECOVERY_STEP; @Configuration -@EnableBatchProcessing public class SSAccessionRecoveryStepConfiguration { @Autowired @Qualifier(SS_ACCESSION_RECOVERY_SERVICE) diff --git a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/SubsnpAccessionsStepConfiguration.java b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/SubsnpAccessionsStepConfiguration.java index 8f4902761..9e2c2db94 100644 --- a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/SubsnpAccessionsStepConfiguration.java +++ b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/SubsnpAccessionsStepConfiguration.java @@ -18,7 +18,6 @@ import org.springframework.batch.core.Step; import org.springframework.batch.core.StepExecutionListener; -import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.batch.core.repository.JobRepository; import org.springframework.batch.core.step.builder.StepBuilder; import org.springframework.batch.core.step.tasklet.TaskletStep; @@ -37,6 +36,7 @@ import uk.ac.ebi.eva.commons.core.models.IVariant; import uk.ac.ebi.eva.commons.core.models.pipeline.Variant; +import static uk.ac.ebi.eva.accession.core.configuration.InMemoryBatchConfiguration.BATCH_TRANSACTION_MANAGER; import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.ACCESSION_WRITER; import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.COMPOSITE_VARIANT_PROCESSOR; import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.SUBSNP_ACCESSION_STEP; @@ -44,7 +44,6 @@ import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.VARIANT_READER; @Configuration -@EnableBatchProcessing public class SubsnpAccessionsStepConfiguration { @Autowired @@ -67,7 +66,8 @@ public class SubsnpAccessionsStepConfiguration { private InvalidVariantSkipPolicy invalidVariantSkipPolicy; @Bean(SUBSNP_ACCESSION_STEP) - public Step subsnpAccessionStep(JobRepository jobRepository, PlatformTransactionManager transactionManager, + public Step subsnpAccessionStep(JobRepository jobRepository, + @Qualifier(BATCH_TRANSACTION_MANAGER) PlatformTransactionManager transactionManager, SimpleCompletionPolicy chunkSizeCompletionPolicy) { TaskletStep step = new StepBuilder(SUBSNP_ACCESSION_STEP, jobRepository) .chunk(chunkSizeCompletionPolicy, transactionManager) diff --git a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/parameters/InputParameters.java b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/parameters/InputParameters.java index 555481c7c..1f6e017e0 100644 --- a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/parameters/InputParameters.java +++ b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/parameters/InputParameters.java @@ -42,8 +42,6 @@ public class InputParameters { private int chunkSize; - private boolean forceRestart; - private ContigNaming contigNaming; private String duplicateSSAccFile; @@ -143,14 +141,6 @@ public void setChunkSize(int chunkSize) { this.chunkSize = chunkSize; } - public boolean isForceRestart() { - return forceRestart; - } - - public void setForceRestart(boolean forceRestart) { - this.forceRestart = forceRestart; - } - public ContigNaming getContigNaming() { return contigNaming; } diff --git a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/runner/EvaAccessionJobLauncherCommandLineRunner.java b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/runner/EvaAccessionJobLauncherCommandLineRunner.java index adcc1202c..da28d1609 100644 --- a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/runner/EvaAccessionJobLauncherCommandLineRunner.java +++ b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/runner/EvaAccessionJobLauncherCommandLineRunner.java @@ -18,7 +18,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.batch.core.Job; -import org.springframework.batch.core.JobExecution; import org.springframework.batch.core.JobExecutionException; import org.springframework.batch.core.JobParameters; import org.springframework.batch.core.JobParametersInvalidException; @@ -35,11 +34,9 @@ import org.springframework.context.ApplicationEventPublisherAware; import org.springframework.stereotype.Component; import org.springframework.util.PatternMatchUtils; -import uk.ac.ebi.eva.accession.core.runner.CommandLineRunnerUtils; import uk.ac.ebi.eva.accession.pipeline.parameters.InputParameters; import uk.ac.ebi.eva.commons.batch.exception.NoJobToExecuteException; import uk.ac.ebi.eva.commons.batch.exception.NoParametersHaveBeenPassedException; -import uk.ac.ebi.eva.commons.batch.exception.NoPreviousJobExecutionException; import uk.ac.ebi.eva.commons.batch.exception.UnknownJobException; import uk.ac.ebi.eva.commons.batch.job.JobExecutionApplicationListener; import uk.ac.ebi.eva.commons.batch.job.JobStatusManager; @@ -120,17 +117,10 @@ public void run(String... args) throws JobExecutionException { JobStatusManager.checkIfJobNameHasBeenDefined(jobName); JobStatusManager.checkIfPropertiesHaveBeenProvided(jobParameters); - if (inputParameters.isForceRestart()) { - JobExecution previousJobExecution = CommandLineRunnerUtils.getLastJobExecution(jobName, jobExplorer, - jobParameters); - CommandLineRunnerUtils.markPreviousJobAsFailed(jobName, jobRepository, - previousJobExecution != null ? previousJobExecution.getJobParameters() : jobParameters); - } else { - jobParameters = CommandLineRunnerUtils.addRunIDToJobParameters(jobName, jobExplorer, jobParameters); - } + launchJob(jobParameters); - } catch (NoJobToExecuteException | NoParametersHaveBeenPassedException | NoPreviousJobExecutionException - | UnknownJobException | JobParametersInvalidException | JobExecutionAlreadyRunningException e) { + } catch (NoJobToExecuteException | NoParametersHaveBeenPassedException | UnknownJobException + | JobParametersInvalidException | JobExecutionAlreadyRunningException e) { logger.error(e.getMessage()); logger.debug("Error trace", e); abnormalExit = true; diff --git a/eva-accession-pipeline/src/main/resources/application.properties b/eva-accession-pipeline/src/main/resources/application.properties index 2b04016ec..135646cf8 100644 --- a/eva-accession-pipeline/src/main/resources/application.properties +++ b/eva-accession-pipeline/src/main/resources/application.properties @@ -26,12 +26,6 @@ spring.data.mongodb.password= spring.data.mongodb.authentication-database= mongodb.read-preference=secondaryPreferred -spring.datasource.driver-class-name=org.postgresql.Driver -spring.datasource.url= -spring.datasource.username= -spring.datasource.password= -spring.datasource.tomcat.max-active=3 - # Only to set up the database! # spring.jpa.generate-ddl=true diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/batch/io/AccessionWriterTest.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/batch/io/AccessionWriterTest.java index ab278fe38..d281e9e7f 100644 --- a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/batch/io/AccessionWriterTest.java +++ b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/batch/io/AccessionWriterTest.java @@ -25,15 +25,17 @@ import org.springframework.batch.item.Chunk; import org.springframework.batch.item.ExecutionContext; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.core.io.ClassPathResource; import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.jdbc.datasource.init.ResourceDatabasePopulator; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.ampt2d.commons.accession.core.models.AccessionWrapper; import uk.ac.ebi.eva.accession.core.batch.io.FastaSequenceReader; +import uk.ac.ebi.eva.accession.core.configuration.ContiguousIdBlocksDataSourceConfiguration; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.SubmittedVariantAccessioningConfiguration; import uk.ac.ebi.eva.accession.core.contig.ContigMapping; import uk.ac.ebi.eva.accession.core.contig.ContigNaming; @@ -43,17 +45,19 @@ import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; import uk.ac.ebi.eva.accession.core.service.GetOrCreateAccessionWrapperCreator; import uk.ac.ebi.eva.accession.core.service.nonhuman.SubmittedVariantAccessioningService; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.core.utils.PipelineTemporaryFolderUtil; import uk.ac.ebi.eva.accession.pipeline.batch.processors.VariantConverter; import uk.ac.ebi.eva.accession.pipeline.configuration.InputParametersConfiguration; import uk.ac.ebi.eva.accession.pipeline.configuration.batch.listeners.ListenersConfiguration; import uk.ac.ebi.eva.accession.pipeline.metric.AccessioningMetric; -import uk.ac.ebi.eva.accession.pipeline.test.MongoTestConfiguration; +import uk.ac.ebi.eva.accession.pipeline.test.BatchTestConfiguration; import uk.ac.ebi.eva.commons.core.models.pipeline.Variant; import uk.ac.ebi.eva.commons.core.models.pipeline.VariantSourceEntry; import uk.ac.ebi.eva.metrics.metric.MetricCompute; +import javax.sql.DataSource; import java.io.BufferedReader; import java.io.File; import java.io.FileInputStream; @@ -61,6 +65,8 @@ import java.io.InputStreamReader; import java.nio.file.Path; import java.nio.file.Paths; +import java.sql.Connection; +import java.sql.Statement; import java.time.LocalDateTime; import java.util.ArrayList; import java.util.Arrays; @@ -77,9 +83,9 @@ import static uk.ac.ebi.eva.accession.pipeline.batch.processors.ContigToGenbankReplacerProcessor.ORIGINAL_CHROMOSOME; @ExtendWith(SpringExtension.class) -@EnableAutoConfiguration @ContextConfiguration(classes = {SubmittedVariantAccessioningConfiguration.class, ListenersConfiguration.class, - InputParametersConfiguration.class, MongoTestConfiguration.class}) + InputParametersConfiguration.class, MongoTestConfiguration.class, ContiguousIdBlocksDataSourceConfiguration.class, +BatchTestConfiguration.class}) @TestPropertySource("classpath:accession-pipeline-test.properties") public class AccessionWriterTest extends MongoTestContainerHelper { @@ -136,6 +142,9 @@ public class AccessionWriterTest extends MongoTestContainerHelper { @Autowired private MetricCompute metricCompute; + @Autowired + private DataSource dataSource; + private ContigMapping contigMapping; @MockBean @@ -153,6 +162,14 @@ public class AccessionWriterTest extends MongoTestContainerHelper { @BeforeEach public void setUp() throws Exception { + try (Connection conn = dataSource.getConnection(); + Statement stmt = conn.createStatement()) { + stmt.execute("DROP TABLE IF EXISTS contiguous_id_blocks"); + } + ResourceDatabasePopulator populator = new ResourceDatabasePopulator(); + populator.addScript(new ClassPathResource("test-data/contiguous_id_blocks_schema.sql")); + populator.execute(dataSource); + contigMapping = new ContigMapping(Arrays.asList( new ContigSynonyms(CHROMOSOME_1, "assembled-molecule", "1", CONTIG_1, "refseq_1", "chr1", true), new ContigSynonyms(CHROMOSOME_2, "assembled-molecule", "2", CONTIG_2, "refseq_2", "chr2", true))); diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSeveralIntervalsOfSubsnpAccessionsJobConfigurationTest.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSeveralIntervalsOfSubsnpAccessionsJobConfigurationTest.java index b65a1f30f..3c7bb12a1 100644 --- a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSeveralIntervalsOfSubsnpAccessionsJobConfigurationTest.java +++ b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSeveralIntervalsOfSubsnpAccessionsJobConfigurationTest.java @@ -26,9 +26,11 @@ import org.springframework.batch.test.JobLauncherTestUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.core.io.ClassPathResource; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.http.HttpMethod; import org.springframework.http.HttpStatus; +import org.springframework.jdbc.datasource.init.ResourceDatabasePopulator; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; @@ -36,18 +38,19 @@ import org.springframework.test.web.client.ExpectedCount; import org.springframework.test.web.client.MockRestServiceServer; import org.springframework.web.client.RestTemplate; +import uk.ac.ebi.eva.accession.core.configuration.ContiguousIdBlocksDataSourceConfiguration; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.SubmittedVariantAccessioningConfiguration; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; import uk.ac.ebi.eva.accession.core.repository.nonhuman.eva.SubmittedVariantAccessioningRepository; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.pipeline.batch.io.AccessionReportWriter; import uk.ac.ebi.eva.accession.pipeline.parameters.InputParameters; -import uk.ac.ebi.eva.accession.pipeline.test.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.accession.pipeline.test.BatchTestConfiguration; -import uk.ac.ebi.eva.accession.pipeline.test.MongoTestConfiguration; import uk.ac.ebi.eva.commons.core.utils.FileUtils; import uk.ac.ebi.eva.metrics.count.CountServiceParameters; +import javax.sql.DataSource; import java.io.BufferedReader; import java.io.FileInputStream; import java.io.IOException; @@ -55,6 +58,8 @@ import java.net.URI; import java.nio.file.Files; import java.nio.file.Paths; +import java.sql.Connection; +import java.sql.Statement; import java.util.Arrays; import java.util.Collection; import java.util.Iterator; @@ -72,7 +77,7 @@ @ExtendWith(SpringExtension.class) @ContextConfiguration(classes = {BatchTestConfiguration.class, SubmittedVariantAccessioningConfiguration.class, - MongoTestConfiguration.class, BatchJobRepositoryTestConfiguration.class}) + MongoTestConfiguration.class, ContiguousIdBlocksDataSourceConfiguration.class}) @TestPropertySource("classpath:accession-pipeline-interval-test.properties") public class CreateSeveralIntervalsOfSubsnpAccessionsJobConfigurationTest extends MongoTestContainerHelper { @@ -94,6 +99,9 @@ public class CreateSeveralIntervalsOfSubsnpAccessionsJobConfigurationTest extend @Autowired private CountServiceParameters countServiceParameters; + @Autowired + private DataSource dataSource; + private final String URL_PATH_SAVE_COUNT = "/v1/bulk/count"; @Autowired @@ -110,6 +118,14 @@ public void setUp() throws Exception { mockServer.expect(ExpectedCount.manyTimes(), requestTo(new URI(countServiceParameters.getUrl() + URL_PATH_SAVE_COUNT))) .andExpect(method(HttpMethod.POST)) .andRespond(withStatus(HttpStatus.OK)); + + try (Connection conn = dataSource.getConnection(); + Statement stmt = conn.createStatement()) { + stmt.execute("DROP TABLE IF EXISTS contiguous_id_blocks"); + } + ResourceDatabasePopulator populator = new ResourceDatabasePopulator(); + populator.addScript(new ClassPathResource("test-data/contiguous_id_blocks_schema.sql")); + populator.execute(dataSource); } @AfterEach diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSubsnpAccessionsJobConfigurationTest.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSubsnpAccessionsJobConfigurationTest.java index 1673ca54a..d5b138d85 100644 --- a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSubsnpAccessionsJobConfigurationTest.java +++ b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSubsnpAccessionsJobConfigurationTest.java @@ -26,9 +26,11 @@ import org.springframework.batch.test.JobLauncherTestUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.core.io.ClassPathResource; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.http.HttpMethod; import org.springframework.http.HttpStatus; +import org.springframework.jdbc.datasource.init.ResourceDatabasePopulator; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; @@ -36,18 +38,19 @@ import org.springframework.test.web.client.ExpectedCount; import org.springframework.test.web.client.MockRestServiceServer; import org.springframework.web.client.RestTemplate; +import uk.ac.ebi.eva.accession.core.configuration.ContiguousIdBlocksDataSourceConfiguration; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.SubmittedVariantAccessioningConfiguration; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; import uk.ac.ebi.eva.accession.core.repository.nonhuman.eva.SubmittedVariantAccessioningRepository; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.pipeline.batch.io.AccessionReportWriter; import uk.ac.ebi.eva.accession.pipeline.parameters.InputParameters; -import uk.ac.ebi.eva.accession.pipeline.test.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.accession.pipeline.test.BatchTestConfiguration; -import uk.ac.ebi.eva.accession.pipeline.test.MongoTestConfiguration; import uk.ac.ebi.eva.commons.core.utils.FileUtils; import uk.ac.ebi.eva.metrics.count.CountServiceParameters; +import javax.sql.DataSource; import java.io.BufferedReader; import java.io.File; import java.io.FileInputStream; @@ -56,6 +59,8 @@ import java.net.URI; import java.nio.file.Files; import java.nio.file.Paths; +import java.sql.Connection; +import java.sql.Statement; import java.util.Collection; import java.util.Iterator; @@ -71,7 +76,7 @@ @ExtendWith(SpringExtension.class) @ContextConfiguration(classes = {BatchTestConfiguration.class, SubmittedVariantAccessioningConfiguration.class, - MongoTestConfiguration.class, BatchJobRepositoryTestConfiguration.class}) + MongoTestConfiguration.class, ContiguousIdBlocksDataSourceConfiguration.class}) @TestPropertySource("classpath:accession-pipeline-test.properties") public class CreateSubsnpAccessionsJobConfigurationTest extends MongoTestContainerHelper { @@ -99,6 +104,9 @@ public class CreateSubsnpAccessionsJobConfigurationTest extends MongoTestContain @Qualifier("COUNT_STATS_REST_TEMPLATE") private RestTemplate restTemplate; + @Autowired + private DataSource dataSource; + private MockRestServiceServer mockServer; @BeforeEach @@ -109,6 +117,14 @@ public void setUp() throws Exception { mockServer.expect(ExpectedCount.manyTimes(), requestTo(new URI(countServiceParameters.getUrl() + URL_PATH_SAVE_COUNT))) .andExpect(method(HttpMethod.POST)) .andRespond(withStatus(HttpStatus.OK)); + + try (Connection conn = dataSource.getConnection(); + Statement stmt = conn.createStatement()) { + stmt.execute("DROP TABLE IF EXISTS contiguous_id_blocks"); + } + ResourceDatabasePopulator populator = new ResourceDatabasePopulator(); + populator.addScript(new ClassPathResource("test-data/contiguous_id_blocks_schema.sql")); + populator.execute(dataSource); } @AfterEach diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSubsnpAccessionsRecoverStateTest.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSubsnpAccessionsRecoverStateTest.java index f1e8b3ad1..200067d5b 100644 --- a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSubsnpAccessionsRecoverStateTest.java +++ b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSubsnpAccessionsRecoverStateTest.java @@ -26,9 +26,11 @@ import org.springframework.batch.test.JobLauncherTestUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.core.io.ClassPathResource; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.http.HttpMethod; import org.springframework.http.HttpStatus; +import org.springframework.jdbc.datasource.init.ResourceDatabasePopulator; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit.jupiter.SpringExtension; @@ -37,24 +39,27 @@ import org.springframework.web.client.RestTemplate; import uk.ac.ebi.ampt2d.commons.accession.persistence.jpa.monotonic.entities.ContiguousIdBlock; import uk.ac.ebi.ampt2d.commons.accession.persistence.jpa.monotonic.repositories.ContiguousIdBlockRepository; +import uk.ac.ebi.eva.accession.core.configuration.ContiguousIdBlocksDataSourceConfiguration; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.SubmittedVariantAccessioningConfiguration; import uk.ac.ebi.eva.accession.core.model.SubmittedVariant; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; import uk.ac.ebi.eva.accession.core.repository.nonhuman.eva.SubmittedVariantAccessioningRepository; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.pipeline.batch.io.AccessionReportWriter; import uk.ac.ebi.eva.accession.pipeline.parameters.InputParameters; -import uk.ac.ebi.eva.accession.pipeline.test.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.accession.pipeline.test.BatchTestConfiguration; -import uk.ac.ebi.eva.accession.pipeline.test.MongoTestConfiguration; import uk.ac.ebi.eva.commons.core.utils.FileUtils; import uk.ac.ebi.eva.metrics.count.CountServiceParameters; +import javax.sql.DataSource; import java.io.FileInputStream; import java.io.IOException; import java.net.URI; import java.nio.file.Files; import java.nio.file.Paths; +import java.sql.Connection; +import java.sql.Statement; import java.util.ArrayList; import java.util.Collection; import java.util.Iterator; @@ -72,7 +77,7 @@ @ExtendWith(SpringExtension.class) @ContextConfiguration(classes = {BatchTestConfiguration.class, SubmittedVariantAccessioningConfiguration.class, - MongoTestConfiguration.class, BatchJobRepositoryTestConfiguration.class}) + MongoTestConfiguration.class, ContiguousIdBlocksDataSourceConfiguration.class}) @TestPropertySource("classpath:accession-pipeline-recover-state-test.properties") public class CreateSubsnpAccessionsRecoverStateTest extends MongoTestContainerHelper { @@ -96,6 +101,9 @@ public class CreateSubsnpAccessionsRecoverStateTest extends MongoTestContainerHe @Qualifier("COUNT_STATS_REST_TEMPLATE") private RestTemplate restTemplate; + @Autowired + private DataSource dataSource; + private final String URL_PATH_SAVE_COUNT = "/v1/bulk/count"; @Autowired @@ -113,6 +121,15 @@ public void setUp() throws Exception { mockServer.expect(ExpectedCount.manyTimes(), requestTo(new URI(countServiceParameters.getUrl() + URL_PATH_SAVE_COUNT))) .andExpect(method(HttpMethod.POST)) .andRespond(withStatus(HttpStatus.OK)); + + try (Connection conn = dataSource.getConnection(); + Statement stmt = conn.createStatement()) { + stmt.execute("DROP TABLE IF EXISTS contiguous_id_blocks"); + } + ResourceDatabasePopulator populator = new ResourceDatabasePopulator(); + populator.addScript(new ClassPathResource("test-data/contiguous_id_blocks_schema.sql")); + populator.addScript(new ClassPathResource("test-data/contiguous_id_blocks_recover_state_data.sql")); + populator.execute(dataSource); } @AfterEach @@ -207,7 +224,7 @@ private void verifyEndDBState() { assertEquals(5000000000l, block1.getFirstValue()); assertEquals(5000000029l, block1.getLastCommitted()); assertEquals(5000000029l, block1.getLastValue()); - assertEquals("0", block1.getApplicationInstanceId()); + assertEquals("1", block1.getApplicationInstanceId()); assertTrue(block1.isNotReserved()); // Block Recovered - (used the 5 unused accessions 5000000035 to 5000000039 and recovered others) @@ -215,7 +232,7 @@ private void verifyEndDBState() { assertEquals(5000000030l, block2.getFirstValue()); assertEquals(5000000059l, block2.getLastCommitted()); assertEquals(5000000059l, block2.getLastValue()); - assertEquals("0", block2.getApplicationInstanceId()); + assertEquals("1", block2.getApplicationInstanceId()); assertTrue(block2.isNotReserved()); // Block Recovered - (No accession used from this block as entire block was already used) @@ -223,7 +240,7 @@ private void verifyEndDBState() { assertEquals(5000000060l, block3.getFirstValue()); assertEquals(5000000089l, block3.getLastCommitted()); assertEquals(5000000089l, block3.getLastValue()); - assertEquals("0", block3.getApplicationInstanceId()); + assertEquals("1", block3.getApplicationInstanceId()); assertTrue(block3.isNotReserved()); // used the remaining 17 (22 - 5 (2nd block)) from 4th block @@ -231,7 +248,7 @@ private void verifyEndDBState() { assertEquals(5000000090l, block4.getFirstValue()); assertEquals(5000000106l, block4.getLastCommitted()); assertEquals(5000000119l, block4.getLastValue()); - assertEquals("0", block4.getApplicationInstanceId()); + assertEquals("1", block4.getApplicationInstanceId()); assertTrue(block4.isNotReserved()); } diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSubsnpAccessionsRecoveringStateJobConfigurationTest.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSubsnpAccessionsRecoveringStateJobConfigurationTest.java index 78ea91fb3..491bf40d6 100644 --- a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSubsnpAccessionsRecoveringStateJobConfigurationTest.java +++ b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSubsnpAccessionsRecoveringStateJobConfigurationTest.java @@ -26,9 +26,11 @@ import org.springframework.batch.test.JobLauncherTestUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.core.io.ClassPathResource; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.http.HttpMethod; import org.springframework.http.HttpStatus; +import org.springframework.jdbc.datasource.init.ResourceDatabasePopulator; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit.jupiter.SpringExtension; @@ -37,24 +39,27 @@ import org.springframework.web.client.RestTemplate; import uk.ac.ebi.ampt2d.commons.accession.persistence.jpa.monotonic.entities.ContiguousIdBlock; import uk.ac.ebi.ampt2d.commons.accession.persistence.jpa.monotonic.repositories.ContiguousIdBlockRepository; +import uk.ac.ebi.eva.accession.core.configuration.ContiguousIdBlocksDataSourceConfiguration; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.SubmittedVariantAccessioningConfiguration; import uk.ac.ebi.eva.accession.core.model.SubmittedVariant; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; import uk.ac.ebi.eva.accession.core.repository.nonhuman.eva.SubmittedVariantAccessioningRepository; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.pipeline.batch.io.AccessionReportWriter; import uk.ac.ebi.eva.accession.pipeline.parameters.InputParameters; -import uk.ac.ebi.eva.accession.pipeline.test.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.accession.pipeline.test.BatchTestConfiguration; -import uk.ac.ebi.eva.accession.pipeline.test.MongoTestConfiguration; import uk.ac.ebi.eva.commons.core.utils.FileUtils; import uk.ac.ebi.eva.metrics.count.CountServiceParameters; +import javax.sql.DataSource; import java.io.FileInputStream; import java.io.IOException; import java.net.URI; import java.nio.file.Files; import java.nio.file.Paths; +import java.sql.Connection; +import java.sql.Statement; import java.util.Collection; import java.util.Iterator; @@ -69,7 +74,7 @@ @ExtendWith(SpringExtension.class) @ContextConfiguration(classes = {BatchTestConfiguration.class, SubmittedVariantAccessioningConfiguration.class, - MongoTestConfiguration.class, BatchJobRepositoryTestConfiguration.class}) + MongoTestConfiguration.class, ContiguousIdBlocksDataSourceConfiguration.class}) @TestPropertySource("classpath:accession-pipeline-recover-test.properties") public class CreateSubsnpAccessionsRecoveringStateJobConfigurationTest extends MongoTestContainerHelper { @@ -100,6 +105,9 @@ public class CreateSubsnpAccessionsRecoveringStateJobConfigurationTest extends M @Autowired private MongoTemplate mongoTemplate; + @Autowired + private DataSource dataSource; + @Autowired @Qualifier(JOB_LAUNCHER_SUBSNP_ACCESSION_JOB) private JobLauncherTestUtils jobLauncherTestUtils; @@ -112,6 +120,16 @@ public void setUp() throws Exception { mockServer.expect(ExpectedCount.manyTimes(), requestTo(new URI(countServiceParameters.getUrl() + URL_PATH_SAVE_COUNT))) .andExpect(method(HttpMethod.POST)) .andRespond(withStatus(HttpStatus.OK)); + + try (Connection conn = dataSource.getConnection(); + Statement stmt = conn.createStatement()) { + stmt.execute("DROP TABLE IF EXISTS contiguous_id_blocks"); + } + + ResourceDatabasePopulator populator = new ResourceDatabasePopulator(); + populator.addScript(new ClassPathResource("test-data/contiguous_id_blocks_schema.sql")); + populator.addScript(new ClassPathResource("test-data/contiguous_id_blocks_data.sql")); + populator.execute(dataSource); } @AfterEach diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/DuplicateSSAccQCJobConfigurationTest.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/DuplicateSSAccQCJobConfigurationTest.java index 8d40527a4..a007e3a30 100644 --- a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/DuplicateSSAccQCJobConfigurationTest.java +++ b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/DuplicateSSAccQCJobConfigurationTest.java @@ -28,12 +28,12 @@ import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit.jupiter.SpringExtension; +import uk.ac.ebi.eva.accession.core.configuration.ContiguousIdBlocksDataSourceConfiguration; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpSubmittedVariantEntity; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; -import uk.ac.ebi.eva.accession.pipeline.test.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.accession.pipeline.test.BatchTestConfiguration; -import uk.ac.ebi.eva.accession.pipeline.test.MongoTestConfiguration; import java.io.IOException; import java.nio.file.Files; @@ -49,7 +49,7 @@ @ExtendWith(SpringExtension.class) @ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class, - BatchJobRepositoryTestConfiguration.class}) + ContiguousIdBlocksDataSourceConfiguration.class}) @TestPropertySource("classpath:duplicate-ss-acc-qc-test.properties") public class DuplicateSSAccQCJobConfigurationTest extends MongoTestContainerHelper { private static final String duplicateSsAccFile = "src/test/resources/duplicateSSAcc.csv"; diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/QCSubsnpAccessionsJobConfigurationTest.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/QCSubsnpAccessionsJobConfigurationTest.java index 9bf30a7ba..5a5de95cd 100644 --- a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/QCSubsnpAccessionsJobConfigurationTest.java +++ b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/QCSubsnpAccessionsJobConfigurationTest.java @@ -29,11 +29,11 @@ import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit.jupiter.SpringExtension; +import uk.ac.ebi.eva.accession.core.configuration.ContiguousIdBlocksDataSourceConfiguration; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.SubmittedVariantAccessioningConfiguration; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; -import uk.ac.ebi.eva.accession.pipeline.test.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.accession.pipeline.test.BatchTestConfiguration; -import uk.ac.ebi.eva.accession.pipeline.test.MongoTestConfiguration; import java.util.Collection; @@ -43,7 +43,7 @@ @ExtendWith(SpringExtension.class) @ContextConfiguration(classes = {BatchTestConfiguration.class, SubmittedVariantAccessioningConfiguration.class, - MongoTestConfiguration.class, BatchJobRepositoryTestConfiguration.class}) + MongoTestConfiguration.class, ContiguousIdBlocksDataSourceConfiguration.class}) @TestPropertySource("classpath:qc-accession-pipeline-test.properties") public class QCSubsnpAccessionsJobConfigurationTest extends MongoTestContainerHelper { @Autowired diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/CreateSubsnpAccessionsStepConfigurationTest.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/CreateSubsnpAccessionsStepConfigurationTest.java index 267a39dba..9dfe92d7d 100644 --- a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/CreateSubsnpAccessionsStepConfigurationTest.java +++ b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/CreateSubsnpAccessionsStepConfigurationTest.java @@ -29,28 +29,33 @@ import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.boot.test.mock.mockito.SpyBean; +import org.springframework.core.io.ClassPathResource; import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.jdbc.datasource.init.ResourceDatabasePopulator; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.ampt2d.commons.accession.persistence.jpa.monotonic.service.ContiguousIdBlockService; +import uk.ac.ebi.eva.accession.core.configuration.ContiguousIdBlocksDataSourceConfiguration; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.SubmittedVariantAccessioningConfiguration; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; import uk.ac.ebi.eva.accession.core.repository.nonhuman.eva.SubmittedVariantAccessioningRepository; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.pipeline.batch.io.AccessionReportWriter; import uk.ac.ebi.eva.accession.pipeline.batch.io.AccessionWriter; import uk.ac.ebi.eva.accession.pipeline.parameters.InputParameters; -import uk.ac.ebi.eva.accession.pipeline.test.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.accession.pipeline.test.BatchTestConfiguration; -import uk.ac.ebi.eva.accession.pipeline.test.MongoTestConfiguration; import uk.ac.ebi.eva.commons.core.utils.FileUtils; +import javax.sql.DataSource; import java.io.FileInputStream; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Paths; +import java.sql.Connection; +import java.sql.Statement; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.Mockito.when; @@ -59,7 +64,7 @@ @ExtendWith(SpringExtension.class) @ContextConfiguration(classes = {BatchTestConfiguration.class, SubmittedVariantAccessioningConfiguration.class, - MongoTestConfiguration.class, BatchJobRepositoryTestConfiguration.class}) + MongoTestConfiguration.class, ContiguousIdBlocksDataSourceConfiguration.class}) @TestPropertySource("classpath:accession-pipeline-test.properties") public class CreateSubsnpAccessionsStepConfigurationTest extends MongoTestContainerHelper { @@ -83,6 +88,9 @@ public class CreateSubsnpAccessionsStepConfigurationTest extends MongoTestContai @Autowired private InputParameters inputParameters; + @Autowired + private DataSource dataSource; + @Autowired private MongoTemplate mongoTemplate; @@ -91,6 +99,14 @@ public class CreateSubsnpAccessionsStepConfigurationTest extends MongoTestContai @BeforeEach public void setUp() throws Exception { + try (Connection conn = dataSource.getConnection(); + Statement stmt = conn.createStatement()) { + stmt.execute("DROP TABLE IF EXISTS contiguous_id_blocks"); + } + ResourceDatabasePopulator populator = new ResourceDatabasePopulator(); + populator.addScript(new ClassPathResource("test-data/contiguous_id_blocks_schema.sql")); + populator.execute(dataSource); + // if a new transaction is not created it will fail when(contiguousIdBlockService.getBlockParameters("test-pipeline-ss")) .thenThrow(RuntimeException.class) diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/QCSubsnpAccessionsStepConfigurationTest.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/QCSubsnpAccessionsStepConfigurationTest.java index 1737d3d62..ac14e6f9a 100644 --- a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/QCSubsnpAccessionsStepConfigurationTest.java +++ b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/QCSubsnpAccessionsStepConfigurationTest.java @@ -31,12 +31,12 @@ import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit.jupiter.SpringExtension; +import uk.ac.ebi.eva.accession.core.configuration.ContiguousIdBlocksDataSourceConfiguration; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.SubmittedVariantAccessioningConfiguration; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.pipeline.batch.io.AccessionWriter; -import uk.ac.ebi.eva.accession.pipeline.test.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.accession.pipeline.test.BatchTestConfiguration; -import uk.ac.ebi.eva.accession.pipeline.test.MongoTestConfiguration; import static org.junit.jupiter.api.Assertions.assertEquals; import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.QC_SUBSNP_ACCESSION_STEP; @@ -44,7 +44,7 @@ @ExtendWith(SpringExtension.class) @ContextConfiguration(classes = {BatchTestConfiguration.class, SubmittedVariantAccessioningConfiguration.class, - MongoTestConfiguration.class, BatchJobRepositoryTestConfiguration.class}) + MongoTestConfiguration.class, ContiguousIdBlocksDataSourceConfiguration.class}) @TestPropertySource("classpath:qc-accession-pipeline-test.properties") public class QCSubsnpAccessionsStepConfigurationTest extends MongoTestContainerHelper { @Autowired diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/runner/EvaAccessionJobLauncherCommandLineRunnerTest.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/runner/EvaAccessionJobLauncherCommandLineRunnerTest.java index 660cef9cb..2f8d20c8d 100644 --- a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/runner/EvaAccessionJobLauncherCommandLineRunnerTest.java +++ b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/runner/EvaAccessionJobLauncherCommandLineRunnerTest.java @@ -20,63 +20,54 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; -import org.springframework.batch.core.JobInstance; -import org.springframework.batch.core.StepExecution; import org.springframework.batch.core.explore.JobExplorer; import org.springframework.batch.core.repository.JobRepository; import org.springframework.batch.test.JobRepositoryTestUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.core.io.ClassPathResource; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.http.HttpMethod; import org.springframework.http.HttpStatus; -import org.springframework.test.annotation.DirtiesContext; +import org.springframework.jdbc.datasource.init.ResourceDatabasePopulator; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit.jupiter.SpringExtension; import org.springframework.test.web.client.ExpectedCount; import org.springframework.test.web.client.MockRestServiceServer; import org.springframework.web.client.RestTemplate; +import uk.ac.ebi.eva.accession.core.configuration.ContiguousIdBlocksDataSourceConfiguration; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; -import uk.ac.ebi.eva.accession.core.runner.CommandLineRunnerUtils; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.pipeline.parameters.InputParameters; -import uk.ac.ebi.eva.accession.pipeline.test.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.accession.pipeline.test.BatchTestConfiguration; -import uk.ac.ebi.eva.accession.pipeline.test.MongoTestConfiguration; import uk.ac.ebi.eva.commons.batch.io.VcfReader; import uk.ac.ebi.eva.metrics.count.CountServiceParameters; +import javax.sql.DataSource; import java.io.File; import java.net.URI; import java.nio.file.Files; -import java.nio.file.NoSuchFileException; import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.StandardCopyOption; -import java.util.Arrays; -import java.util.Collections; +import java.sql.Connection; +import java.sql.Statement; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.springframework.test.web.client.match.MockRestRequestMatchers.method; import static org.springframework.test.web.client.match.MockRestRequestMatchers.requestTo; import static org.springframework.test.web.client.response.MockRestResponseCreators.withStatus; import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.SUBSNP_ACCESSION_JOB; -import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.SUBSNP_ACCESSION_STEP; import static uk.ac.ebi.eva.accession.pipeline.runner.RunnerUtil.deleteTemporaryContigAndVariantFiles; import static uk.ac.ebi.eva.accession.pipeline.runner.RunnerUtil.getOriginalVcfContent; -import static uk.ac.ebi.eva.accession.pipeline.runner.RunnerUtil.injectErrorIntoTempVcf; -import static uk.ac.ebi.eva.accession.pipeline.runner.RunnerUtil.remediateTempVcfError; import static uk.ac.ebi.eva.accession.pipeline.runner.RunnerUtil.useOriginalVcfFile; -import static uk.ac.ebi.eva.accession.pipeline.runner.RunnerUtil.useTempVcfFile; import static uk.ac.ebi.eva.accession.pipeline.runner.RunnerUtil.writeToTempVCFFile; @ExtendWith(SpringExtension.class) @ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class, - BatchJobRepositoryTestConfiguration.class}) + ContiguousIdBlocksDataSourceConfiguration.class}) @TestPropertySource("classpath:accession-pipeline-test.properties") public class EvaAccessionJobLauncherCommandLineRunnerTest extends MongoTestContainerHelper { @@ -117,12 +108,13 @@ public class EvaAccessionJobLauncherCommandLineRunnerTest extends MongoTestConta private static String originalVcfInputFilePath; - private static String originalVcfOutputFilePath; - private static String originalVcfContent; private boolean originalInputParametersCaptured = false; + @Autowired + private DataSource dataSource; + @BeforeAll public static void initializeTempFile() throws Exception { tempVcfInputFileToTestFailingJobs = File.createTempFile("resumeFailingJob", ".vcf.gz"); @@ -138,7 +130,6 @@ public static void deleteTempFile() { public void setUp() throws Exception { if (!originalInputParametersCaptured) { originalVcfInputFilePath = inputParameters.getVcf(); - originalVcfOutputFilePath = inputParameters.getOutputVcf(); originalVcfContent = getOriginalVcfContent(originalVcfInputFilePath); writeToTempVCFFile(originalVcfContent, tempVcfInputFileToTestFailingJobs); originalInputParametersCaptured = true; @@ -154,12 +145,19 @@ public void setUp() throws Exception { .andRespond(withStatus(HttpStatus.OK)); mongoTemplate.dropCollection(SubmittedVariantEntity.class); + + try (Connection conn = dataSource.getConnection(); + Statement stmt = conn.createStatement()) { + stmt.execute("DROP TABLE IF EXISTS contiguous_id_blocks"); + } + ResourceDatabasePopulator populator = new ResourceDatabasePopulator(); + populator.addScript(new ClassPathResource("test-data/contiguous_id_blocks_schema.sql")); + populator.execute(dataSource); } @AfterEach public void tearDown() { jobRepositoryTestUtils.removeJobExecutions(); - inputParameters.setForceRestart(false); } @Test @@ -176,129 +174,4 @@ public void runJobWithNoName() throws Exception { assertEquals(EvaAccessionJobLauncherCommandLineRunner.EXIT_WITH_ERRORS, runner.getExitCode()); } - - @Test - @DirtiesContext - @Disabled - public void restartCompletedJobThatIsAlreadyInTheRepository() throws Exception { - runner.run(); - assertEquals(EvaAccessionJobLauncherCommandLineRunner.EXIT_WITHOUT_ERRORS, runner.getExitCode()); - - deleteTemporaryContigAndVariantFiles(inputParameters, tempVcfOutputDir); - - inputParameters.setForceRestart(true); - runner.run(); - assertEquals(EvaAccessionJobLauncherCommandLineRunner.EXIT_WITHOUT_ERRORS, runner.getExitCode()); - } - - - private JobInstance runJobAandCheckResults() throws Exception { - runner.run(); - assertEquals(EvaAccessionJobLauncherCommandLineRunner.EXIT_WITH_ERRORS, runner.getExitCode()); - JobInstance currentJobInstance = CommandLineRunnerUtils.getLastJobExecution(SUBSNP_ACCESSION_JOB, - jobExplorer, - inputParameters.toJobParameters()) - .getJobInstance(); - StepExecution stepExecution = jobRepository.getLastStepExecution(currentJobInstance, SUBSNP_ACCESSION_STEP); - //Ensure that only the first batch was written (batch size is 5 and error was at line#9) - assertEquals(inputParameters.getChunkSize(), stepExecution.getWriteCount()); - - return currentJobInstance; - } - - @Test - @DirtiesContext - public void forceRestartButNoJobInTheRepository() throws Exception { - inputParameters.setForceRestart(true); - assertEquals(Collections.EMPTY_LIST, jobExplorer.getJobNames()); - runner.run(); - - assertEquals(EvaAccessionJobLauncherCommandLineRunner.EXIT_WITHOUT_ERRORS, runner.getExitCode()); - } - - @Test - @DirtiesContext - @Disabled - public void resumeFailingJobFromCorrectChunk() throws Exception { - // Jobs A, B, C are run chronological order; A and C have SAME parameters; - // A is the job that is run after VCF fault injection (as part of the runTestWithFaultInjection method), - // therefore should fail. - // B is a job run with the original VCF without any faults (run separately), therefore should succeed. - // C is a job with the same parameters as A run after VCF fault remediation (as part of the - // runTestWithFaultInjection method), therefore should resume A and succeed. - - useTempVcfFile(inputParameters, tempVcfInputFileToTestFailingJobs, vcfReader); - String modifiedVcfContent = originalVcfContent.replace("76852", "76852jibberish"); - injectErrorIntoTempVcf(modifiedVcfContent, tempVcfInputFileToTestFailingJobs); - JobInstance failingJobInstance = runJobAandCheckResults(); - - runJobBAndCheckResults(); - - remediateTempVcfError(originalVcfContent, tempVcfInputFileToTestFailingJobs); - runJobCAndCheckResumption(failingJobInstance); - } - - private void runJobBAndCheckResults() throws Exception { - useOriginalVcfFile(inputParameters, originalVcfInputFilePath, vcfReader); - // Back up contig and variant files (left behind by previous unsuccessful job A) to temp folder - // so as to not interfere with this job's execution which uses the original VCF file - backUpContigAndVariantFilesToTempFolder(); - - runner.run(); - assertEquals(EvaAccessionJobLauncherCommandLineRunner.EXIT_WITHOUT_ERRORS, runner.getExitCode()); - - //Restore state so that Job C can continue running after fault remediation - useTempVcfFile(inputParameters, tempVcfInputFileToTestFailingJobs, vcfReader); - restoreContigAndVariantFilesFromTempFolder(); - } - - private void runJobCAndCheckResumption(JobInstance failingJobInstance) throws Exception { - runner.run(); - JobInstance currentJobInstance = CommandLineRunnerUtils.getLastJobExecution(SUBSNP_ACCESSION_JOB, - jobExplorer, - inputParameters.toJobParameters()) - .getJobInstance(); - StepExecution stepExecution = jobRepository.getLastStepExecution(currentJobInstance, SUBSNP_ACCESSION_STEP); - // Did we resume the previous failed job instance? - assertEquals(failingJobInstance.getInstanceId(), currentJobInstance.getInstanceId()); - - int numberOfLinesInVcf = getNumberOfLinesInVcfString(originalVcfContent); - int numberOfNonVariants = 1; //TBGI000010 is a non-variant - // Test resumption point - did we pick up where we left off? - // Ensure all the batches other than the first batch were processed - assertEquals(numberOfLinesInVcf - inputParameters.getChunkSize() - numberOfNonVariants, - stepExecution.getWriteCount()); - assertEquals(EvaAccessionJobLauncherCommandLineRunner.EXIT_WITHOUT_ERRORS, runner.getExitCode()); - } - - - private void backUpContigAndVariantFilesToTempFolder() { - moveFile(Paths.get(originalVcfOutputFilePath + ".contigs"), - Paths.get(tempVcfOutputDir + "/accession-output.vcf.contigs")); - moveFile(Paths.get(originalVcfOutputFilePath + ".variants"), - Paths.get(tempVcfOutputDir + "/accession-output.vcf.variants")); - } - - private void restoreContigAndVariantFilesFromTempFolder() { - moveFile(Paths.get(tempVcfOutputDir + "/accession-output.vcf.contigs"), - Paths.get(Paths.get(originalVcfOutputFilePath).getParent() + "/accession-output.vcf.contigs")); - moveFile(Paths.get(tempVcfOutputDir + "/accession-output.vcf.variants"), - Paths.get(Paths.get(originalVcfOutputFilePath).getParent() + "/accession-output.vcf.variants")); - } - - private void moveFile(Path source, Path destination) { - try { - Files.move(source, destination, StandardCopyOption.REPLACE_EXISTING); - } catch (Exception ex) { - if (!(ex instanceof NoSuchFileException)) { - throw new RuntimeException(ex); - } - } - } - - private int getNumberOfLinesInVcfString(String vcfString) { - return (int) Arrays.stream(vcfString.split(System.lineSeparator())) - .filter(line -> !line.startsWith("#")) - .count(); - } } \ No newline at end of file diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/runner/JobFailureBlocksReleasedTest.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/runner/JobFailureBlocksReleasedTest.java index cbbb5eebc..15742b21d 100644 --- a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/runner/JobFailureBlocksReleasedTest.java +++ b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/runner/JobFailureBlocksReleasedTest.java @@ -22,23 +22,28 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.io.ClassPathResource; import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.jdbc.datasource.init.ResourceDatabasePopulator; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.ampt2d.commons.accession.persistence.jpa.monotonic.repositories.ContiguousIdBlockRepository; +import uk.ac.ebi.eva.accession.core.configuration.ContiguousIdBlocksDataSourceConfiguration; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.pipeline.parameters.InputParameters; -import uk.ac.ebi.eva.accession.pipeline.test.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.accession.pipeline.test.BatchTestConfiguration; -import uk.ac.ebi.eva.accession.pipeline.test.MongoTestConfiguration; import uk.ac.ebi.eva.commons.batch.io.VcfReader; +import javax.sql.DataSource; import java.io.File; import java.nio.file.Files; import java.nio.file.Path; +import java.sql.Connection; +import java.sql.Statement; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -52,7 +57,7 @@ @ExtendWith(SpringExtension.class) @ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class, - BatchJobRepositoryTestConfiguration.class}) + ContiguousIdBlocksDataSourceConfiguration.class}) @TestPropertySource("classpath:accession-pipeline-test.properties") public class JobFailureBlocksReleasedTest extends MongoTestContainerHelper { @Autowired @@ -78,6 +83,9 @@ public class JobFailureBlocksReleasedTest extends MongoTestContainerHelper { @Autowired private ContiguousIdBlockRepository blockRepository; + @Autowired + private DataSource dataSource; + @BeforeAll public static void initializeTempFile() throws Exception { tempVcfInputFileToTestFailingJobs = File.createTempFile("resumeFailingJob", ".vcf.gz"); @@ -100,6 +108,15 @@ public void setUp() throws Exception { useOriginalVcfFile(inputParameters, originalVcfInputFilePath, vcfReader); mongoTemplate.dropCollection(SubmittedVariantEntity.class); + + try (Connection conn = dataSource.getConnection(); + Statement stmt = conn.createStatement()) { + stmt.execute("DROP TABLE IF EXISTS contiguous_id_blocks"); + } + ResourceDatabasePopulator populator = new ResourceDatabasePopulator(); + populator.addScript(new ClassPathResource("test-data/contiguous_id_blocks_schema.sql")); + populator.execute(dataSource); + } @Test diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/runner/RestartFailedJobTest.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/runner/RestartFailedJobTest.java deleted file mode 100644 index ea0cd06df..000000000 --- a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/runner/RestartFailedJobTest.java +++ /dev/null @@ -1,214 +0,0 @@ -/* - * Copyright 2014-2018 EMBL - European Bioinformatics Institute - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package uk.ac.ebi.eva.accession.pipeline.runner; - -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.springframework.batch.core.JobInstance; -import org.springframework.batch.core.StepExecution; -import org.springframework.batch.core.explore.JobExplorer; -import org.springframework.batch.core.repository.JobRepository; -import org.springframework.batch.test.JobRepositoryTestUtils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.boot.test.mock.mockito.SpyBean; -import org.springframework.data.mongodb.core.MongoTemplate; -import org.springframework.http.HttpMethod; -import org.springframework.http.HttpStatus; -import org.springframework.test.annotation.DirtiesContext; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit.jupiter.SpringExtension; -import org.springframework.test.web.client.ExpectedCount; -import org.springframework.test.web.client.MockRestServiceServer; -import org.springframework.web.client.RestTemplate; -import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; -import uk.ac.ebi.eva.accession.core.runner.CommandLineRunnerUtils; -import uk.ac.ebi.eva.accession.core.service.nonhuman.SubmittedVariantAccessioningService; -import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; -import uk.ac.ebi.eva.accession.pipeline.parameters.InputParameters; -import uk.ac.ebi.eva.accession.pipeline.test.BatchJobRepositoryTestConfiguration; -import uk.ac.ebi.eva.accession.pipeline.test.BatchTestConfiguration; -import uk.ac.ebi.eva.accession.pipeline.test.MongoTestConfiguration; -import uk.ac.ebi.eva.commons.batch.io.VcfReader; -import uk.ac.ebi.eva.metrics.count.CountServiceParameters; - -import java.io.File; -import java.net.URI; -import java.nio.file.Files; -import java.nio.file.Path; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotEquals; -import static org.mockito.Mockito.doNothing; -import static org.springframework.test.web.client.match.MockRestRequestMatchers.method; -import static org.springframework.test.web.client.match.MockRestRequestMatchers.requestTo; -import static org.springframework.test.web.client.response.MockRestResponseCreators.withStatus; -import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.SUBSNP_ACCESSION_JOB; -import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.SUBSNP_ACCESSION_STEP; -import static uk.ac.ebi.eva.accession.pipeline.runner.RunnerUtil.deleteTemporaryContigAndVariantFiles; -import static uk.ac.ebi.eva.accession.pipeline.runner.RunnerUtil.getOriginalVcfContent; -import static uk.ac.ebi.eva.accession.pipeline.runner.RunnerUtil.injectErrorIntoTempVcf; -import static uk.ac.ebi.eva.accession.pipeline.runner.RunnerUtil.remediateTempVcfError; -import static uk.ac.ebi.eva.accession.pipeline.runner.RunnerUtil.useOriginalVcfFile; -import static uk.ac.ebi.eva.accession.pipeline.runner.RunnerUtil.useTempVcfFile; -import static uk.ac.ebi.eva.accession.pipeline.runner.RunnerUtil.writeToTempVCFFile; - -@ExtendWith(SpringExtension.class) -@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class, - BatchJobRepositoryTestConfiguration.class}) -@TestPropertySource("classpath:accession-pipeline-test.properties") -public class RestartFailedJobTest extends MongoTestContainerHelper { - - @Autowired - private InputParameters inputParameters; - - @Autowired - private JobRepository jobRepository; - - @Autowired - private JobExplorer jobExplorer; - - @Autowired - private EvaAccessionJobLauncherCommandLineRunner runner; - - @Autowired - private VcfReader vcfReader; - - @Autowired - private CountServiceParameters countServiceParameters; - - @Autowired - @Qualifier("COUNT_STATS_REST_TEMPLATE") - private RestTemplate restTemplate; - - @Autowired - private MongoTemplate mongoTemplate; - - private final String URL_PATH_SAVE_COUNT = "/v1/bulk/count"; - - private MockRestServiceServer mockServer; - - private JobRepositoryTestUtils jobRepositoryTestUtils; - - private static File tempVcfInputFileToTestFailingJobs; - - private static Path tempVcfOutputDir; - - private static String originalVcfInputFilePath; - - private static String originalVcfContent; - - private boolean originalInputParametersCaptured = false; - - @SpyBean - private SubmittedVariantAccessioningService accessioningServiceSpy; - - @BeforeAll - public static void initializeTempFile() throws Exception { - tempVcfInputFileToTestFailingJobs = File.createTempFile("resumeFailingJob", ".vcf.gz"); - tempVcfOutputDir = Files.createTempDirectory("contigs_variants_dir"); - } - - @AfterAll - public static void deleteTempFile() { - tempVcfInputFileToTestFailingJobs.delete(); - } - - @BeforeEach - public void setUp() throws Exception { - if (!originalInputParametersCaptured) { - originalVcfInputFilePath = inputParameters.getVcf(); - originalVcfContent = getOriginalVcfContent(originalVcfInputFilePath); - writeToTempVCFFile(originalVcfContent, tempVcfInputFileToTestFailingJobs); - originalInputParametersCaptured = true; - } - jobRepositoryTestUtils = new JobRepositoryTestUtils(jobRepository); - runner.setJobName(SUBSNP_ACCESSION_JOB); - deleteTemporaryContigAndVariantFiles(inputParameters, tempVcfOutputDir); - useOriginalVcfFile(inputParameters, originalVcfInputFilePath, vcfReader); - - mockServer = MockRestServiceServer.createServer(restTemplate); - mockServer.expect(ExpectedCount.manyTimes(), requestTo(new URI(countServiceParameters.getUrl() + URL_PATH_SAVE_COUNT))) - .andExpect(method(HttpMethod.POST)) - .andRespond(withStatus(HttpStatus.OK)); - - mongoTemplate.dropCollection(SubmittedVariantEntity.class); - // Mock the behavior of shutDownAccessionGenerator method to do nothing - doNothing().when(accessioningServiceSpy).shutDownAccessionGenerator(); - - } - - @AfterEach - public void tearDown() { - jobRepositoryTestUtils.removeJobExecutions(); - inputParameters.setForceRestart(false); - } - - /* - * Separated this test from the rest of the tests in EvaAccessionJobLauncherCommandLineRunnerTest, - * as we have to Mock(Spy to be exact) on the SubmittedVariantAccessioningService bean in order to reuse the same - * without shutting down its accession generator. - * - * Ideally, we should not be Spying but rather the jobs when restarting should be using a new instance of the service. - * but it was tricky to inject, hence the workaround. - * */ - @Test - @DirtiesContext - public void restartFailedJobThatIsAlreadyInTheRepository() throws Exception { - useTempVcfFile(inputParameters, tempVcfInputFileToTestFailingJobs, vcfReader); - String modifiedVcfContent = originalVcfContent.replace("76852", "76852jibberish"); - injectErrorIntoTempVcf(modifiedVcfContent, tempVcfInputFileToTestFailingJobs); - JobInstance failingJobInstance = runJobAandCheckResults(); - - mongoTemplate.dropCollection(SubmittedVariantEntity.class); - - inputParameters.setForceRestart(true); - remediateTempVcfError(originalVcfContent, tempVcfInputFileToTestFailingJobs); - deleteTemporaryContigAndVariantFiles(inputParameters, tempVcfOutputDir); //left behind by unsuccessful job A - runJobBAndCheckRestart(failingJobInstance); - } - - private JobInstance runJobAandCheckResults() throws Exception { - runner.run(); - assertEquals(EvaAccessionJobLauncherCommandLineRunner.EXIT_WITH_ERRORS, runner.getExitCode()); - JobInstance currentJobInstance = CommandLineRunnerUtils.getLastJobExecution(SUBSNP_ACCESSION_JOB, - jobExplorer, - inputParameters.toJobParameters()) - .getJobInstance(); - StepExecution stepExecution = jobRepository.getLastStepExecution(currentJobInstance, SUBSNP_ACCESSION_STEP); - //Ensure that only the first batch was written (batch size is 5 and error was at line#9) - assertEquals(inputParameters.getChunkSize(), stepExecution.getWriteCount()); - - return currentJobInstance; - } - - private void runJobBAndCheckRestart(JobInstance failingJobInstance) throws Exception { - runner.run(); - assertEquals(EvaAccessionJobLauncherCommandLineRunner.EXIT_WITHOUT_ERRORS, runner.getExitCode()); - JobInstance currentJobInstance = CommandLineRunnerUtils.getLastJobExecution(SUBSNP_ACCESSION_JOB, - jobExplorer, - inputParameters.toJobParameters()) - .getJobInstance(); - assertNotEquals(failingJobInstance.getInstanceId(), currentJobInstance.getInstanceId()); - } - -} \ No newline at end of file diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/runner/SSAccessionRecoveryTest.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/runner/SSAccessionRecoveryTest.java index dacbf692c..0fbcdc038 100644 --- a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/runner/SSAccessionRecoveryTest.java +++ b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/runner/SSAccessionRecoveryTest.java @@ -16,24 +16,32 @@ * limitations under the License. */ +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.batch.test.context.SpringBatchTest; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.io.ClassPathResource; +import org.springframework.jdbc.datasource.init.ResourceDatabasePopulator; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.ampt2d.commons.accession.persistence.jpa.monotonic.entities.ContiguousIdBlock; import uk.ac.ebi.ampt2d.commons.accession.persistence.jpa.monotonic.repositories.ContiguousIdBlockRepository; +import uk.ac.ebi.eva.accession.core.configuration.ContiguousIdBlocksDataSourceConfiguration; +import uk.ac.ebi.eva.accession.core.configuration.InMemoryBatchConfiguration; import uk.ac.ebi.eva.accession.core.model.SubmittedVariant; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; import uk.ac.ebi.eva.accession.core.repository.nonhuman.eva.SubmittedVariantAccessioningRepository; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; -import uk.ac.ebi.eva.accession.pipeline.test.BatchJobRepositoryTestConfiguration; -import uk.ac.ebi.eva.accession.pipeline.test.MongoTestConfiguration; +import uk.ac.ebi.eva.accession.pipeline.test.BatchTestConfiguration; import uk.ac.ebi.eva.accession.pipeline.test.SSAccessionRecoveryTestConfiguration; +import javax.sql.DataSource; +import java.sql.Connection; +import java.sql.Statement; import java.util.ArrayList; import java.util.List; @@ -43,7 +51,7 @@ @ExtendWith(SpringExtension.class) @ContextConfiguration(classes = {SSAccessionRecoveryTestConfiguration.class, MongoTestConfiguration.class, - BatchJobRepositoryTestConfiguration.class}) + ContiguousIdBlocksDataSourceConfiguration.class, InMemoryBatchConfiguration.class, BatchTestConfiguration.class}) @TestPropertySource("classpath:ss-accession-recovery.properties") @SpringBatchTest public class SSAccessionRecoveryTest extends MongoTestContainerHelper { @@ -56,6 +64,22 @@ public class SSAccessionRecoveryTest extends MongoTestContainerHelper { @Autowired private SubmittedVariantAccessioningRepository mongoRepository; + @Autowired + private DataSource dataSource; + + @BeforeEach + public void setUp() throws Exception { + try (Connection conn = dataSource.getConnection(); + Statement stmt = conn.createStatement()) { + stmt.execute("DROP TABLE IF EXISTS contiguous_id_blocks"); + } + + ResourceDatabasePopulator populator = new ResourceDatabasePopulator(); + populator.addScript(new ClassPathResource("test-data/contiguous_id_blocks_schema.sql")); + populator.addScript(new ClassPathResource("test-data/ss_accession_recovery_test_data.sql")); + populator.execute(dataSource); + } + @Test @DirtiesContext public void testContiguousBlocksForCategorySSAreRecovered() throws Exception { @@ -192,7 +216,7 @@ private void verifyEndDBState() { assertEquals(5000000000l, block1.getFirstValue()); assertEquals(5000000029l, block1.getLastCommitted()); assertEquals(5000000029l, block1.getLastValue()); - assertEquals("0", block1.getApplicationInstanceId()); + assertEquals("1", block1.getApplicationInstanceId()); assertTrue(block1.isNotReserved()); // Block Recovered partially - (used 5000000030-5000000034 and 5000000040-5000000059) @@ -201,7 +225,7 @@ private void verifyEndDBState() { assertEquals(5000000030l, block2.getFirstValue()); assertEquals(5000000034l, block2.getLastCommitted()); assertEquals(5000000059l, block2.getLastValue()); - assertEquals("0", block2.getApplicationInstanceId()); + assertEquals("1", block2.getApplicationInstanceId()); assertTrue(block2.isNotReserved()); // Block Recovered - recovered entire block @@ -209,7 +233,7 @@ private void verifyEndDBState() { assertEquals(5000000060l, block3.getFirstValue()); assertEquals(5000000089l, block3.getLastCommitted()); assertEquals(5000000089l, block3.getLastValue()); - assertEquals("0", block3.getApplicationInstanceId()); + assertEquals("1", block3.getApplicationInstanceId()); assertTrue(block3.isNotReserved()); // Block Recovered - None of the accessions are used, just released the block @@ -217,7 +241,7 @@ private void verifyEndDBState() { assertEquals(5000000090l, block4.getFirstValue()); assertEquals(5000000089l, block4.getLastCommitted()); assertEquals(5000000119l, block4.getLastValue()); - assertEquals("0", block4.getApplicationInstanceId()); + assertEquals("1", block4.getApplicationInstanceId()); assertTrue(block4.isNotReserved()); // Block Not Recovered - Block not to be recovered as it's last update is after cut off time diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/test/BatchJobRepositoryTestConfiguration.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/test/BatchJobRepositoryTestConfiguration.java deleted file mode 100644 index 776d64a87..000000000 --- a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/test/BatchJobRepositoryTestConfiguration.java +++ /dev/null @@ -1,22 +0,0 @@ -package uk.ac.ebi.eva.accession.pipeline.test; - -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Primary; -import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseBuilder; -import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseType; - -import javax.sql.DataSource; - -@Configuration -public class BatchJobRepositoryTestConfiguration { - - @Bean - @Primary - public DataSource dataSource() { - return new EmbeddedDatabaseBuilder() - .setType(EmbeddedDatabaseType.HSQL) - .addScript("org/springframework/batch/core/schema-hsqldb.sql") - .build(); - } -} \ No newline at end of file diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/test/BatchTestConfiguration.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/test/BatchTestConfiguration.java index 38a6459e5..024862093 100644 --- a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/test/BatchTestConfiguration.java +++ b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/test/BatchTestConfiguration.java @@ -21,9 +21,10 @@ import org.springframework.batch.core.repository.JobRepository; import org.springframework.batch.test.JobLauncherTestUtils; import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Import; +import org.springframework.web.client.RestTemplate; +import uk.ac.ebi.eva.accession.core.configuration.InMemoryBatchConfiguration; import uk.ac.ebi.eva.accession.pipeline.configuration.batch.io.AccessionWriterConfiguration; import uk.ac.ebi.eva.accession.pipeline.configuration.batch.io.DuplicateSSAccQCWriterConfiguration; import uk.ac.ebi.eva.accession.pipeline.configuration.batch.io.SSAccFileReaderConfiguration; @@ -48,8 +49,7 @@ import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.QC_SUBSNP_ACCESSION_JOB; import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.SUBSNP_ACCESSION_JOB; -@EnableAutoConfiguration -@Import({SubsnpAccessionsJobConfiguration.class, SubsnpAccessionsStepConfiguration.class, +@Import({InMemoryBatchConfiguration.class, SubsnpAccessionsJobConfiguration.class, SubsnpAccessionsStepConfiguration.class, QCSubsnpAccessionsJobConfiguration.class, QCSubsnpAccessionsStepConfiguration.class, VcfReaderConfiguration.class, VariantProcessorConfiguration.class, AccessionWriterConfiguration.class, BuildReportStepConfiguration.class, AccessioningShutdownStepConfiguration.class, @@ -97,4 +97,9 @@ public JobLauncherTestUtils jobLauncherTestUtilsDuplicateSSAccQC(JobLauncher job public JobExecutionApplicationListener jobExecutionApplicationListener() { return new JobExecutionApplicationListener(); } + + @Bean(name = "COUNT_STATS_REST_TEMPLATE") + public RestTemplate countStatsRestTemplate() { + return new RestTemplate(); + } } \ No newline at end of file diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/test/MongoTestConfiguration.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/test/MongoTestConfiguration.java deleted file mode 100644 index 2a261f1b6..000000000 --- a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/test/MongoTestConfiguration.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Copyright 2021 EMBL - European Bioinformatics Institute - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package uk.ac.ebi.eva.accession.pipeline.test; - -import com.mongodb.ReadPreference; -import com.mongodb.WriteConcern; -import com.mongodb.client.MongoClient; -import com.mongodb.client.MongoClients; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.domain.EntityScan; -import org.springframework.boot.test.autoconfigure.data.mongo.AutoConfigureDataMongo; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Primary; -import org.springframework.data.mongodb.MongoDatabaseFactory; -import org.springframework.data.mongodb.config.EnableMongoAuditing; -import org.springframework.data.mongodb.core.MongoTemplate; -import org.springframework.data.mongodb.core.SimpleMongoClientDatabaseFactory; -import org.springframework.data.mongodb.core.WriteResultChecking; -import org.springframework.data.mongodb.core.convert.DefaultDbRefResolver; -import org.springframework.data.mongodb.core.convert.DefaultMongoTypeMapper; -import org.springframework.data.mongodb.core.convert.MappingMongoConverter; -import org.springframework.data.mongodb.core.convert.MongoCustomConversions; -import org.springframework.data.mongodb.core.mapping.MongoMappingContext; -import org.springframework.data.mongodb.repository.config.EnableMongoRepositories; - -import java.util.Collections; - -@Configuration -@EntityScan(basePackages = {"uk.ac.ebi.eva.accession.core.persistence"}) -@EnableMongoRepositories(basePackages = { - "uk.ac.ebi.eva.accession.core.persistence", - "uk.ac.ebi.ampt2d.commons.accession.persistence.mongodb.repository"}) -@EnableMongoAuditing -@AutoConfigureDataMongo -public class MongoTestConfiguration { - - @Value("${spring.data.mongodb.uri}") - private String mongoUri; - - @Value("${spring.data.mongodb.database}") - private String database; - - @Value("${mongodb.read-preference:primary}") - private String readPreference; - - @Bean - @Primary - public MongoClient mongoClient() { - return MongoClients.create(mongoUri); - } - - @Bean("primaryFactory") - @Primary - public MongoDatabaseFactory mongoDbFactory() { - return new SimpleMongoClientDatabaseFactory(mongoClient(), database); - } - - @Bean - @Primary - public MongoCustomConversions mongoCustomConversions() { - return new MongoCustomConversions(Collections.emptyList()); - } - - @Bean - @Primary - public MongoMappingContext mongoMappingContext(MongoCustomConversions conversions) { - MongoMappingContext context = new MongoMappingContext(); - context.setSimpleTypeHolder(conversions.getSimpleTypeHolder()); - context.setAutoIndexCreation(false); - return context; - } - - @Bean - @Primary - public MongoTemplate mongoTemplate(MongoMappingContext mongoMappingContext, - MongoCustomConversions conversions) { - MappingMongoConverter converter = new MappingMongoConverter( - new DefaultDbRefResolver(mongoDbFactory()), mongoMappingContext); - converter.setCustomConversions(conversions); - converter.setTypeMapper(new DefaultMongoTypeMapper(null)); - converter.afterPropertiesSet(); - MongoTemplate mongoTemplate = new MongoTemplate(mongoDbFactory(), converter); - mongoTemplate.setReadPreference(ReadPreference.valueOf(readPreference)); - mongoTemplate.setWriteConcern(WriteConcern.MAJORITY); - mongoTemplate.setWriteResultChecking(WriteResultChecking.EXCEPTION); - return mongoTemplate; - } -} diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/test/SSAccessionRecoveryTestConfiguration.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/test/SSAccessionRecoveryTestConfiguration.java index 4556a4439..0adc8b321 100644 --- a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/test/SSAccessionRecoveryTestConfiguration.java +++ b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/test/SSAccessionRecoveryTestConfiguration.java @@ -19,7 +19,6 @@ import org.springframework.batch.core.launch.JobLauncher; import org.springframework.batch.core.repository.JobRepository; import org.springframework.batch.test.JobLauncherTestUtils; -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Import; import uk.ac.ebi.eva.accession.pipeline.configuration.batch.io.AccessionWriterConfiguration; @@ -31,7 +30,6 @@ import uk.ac.ebi.eva.accession.pipeline.runner.EvaAccessionJobLauncherCommandLineRunner; import uk.ac.ebi.eva.commons.batch.job.JobExecutionApplicationListener; -@EnableAutoConfiguration @Import({SSAccessionRecoveryJobConfiguration.class, SSAccessionRecoveryStepConfiguration.class, SSAccessionRecoveryServiceConfiguration.class, diff --git a/eva-accession-pipeline/src/test/resources/properties/accession-pipeline-interval-test.properties b/eva-accession-pipeline/src/test/resources/properties/accession-pipeline-interval-test.properties index b30fb3297..3f84f1df4 100644 --- a/eva-accession-pipeline/src/test/resources/properties/accession-pipeline-interval-test.properties +++ b/eva-accession-pipeline/src/test/resources/properties/accession-pipeline-interval-test.properties @@ -4,6 +4,11 @@ accessioning.monotonic.test-pipeline-ss.blockSize=2 accessioning.monotonic.test-pipeline-ss.blockStartValue=500 accessioning.monotonic.test-pipeline-ss.nextBlockInterval=10 +continuous.id.blocks.datasource.driver-class-name=org.hsqldb.jdbc.JDBCDriver +continuous.id.blocks.datasource.jdbc-url=jdbc:hsqldb:mem:db;sql.syntax_pgs=true;DB_CLOSE_DELAY=-1 +continuous.id.blocks.datasource.username=SA +continuous.id.blocks.datasource.password= + eva.count-stats.url=http://localhost:8080 eva.count-stats.username=username eva.count-stats.password=password diff --git a/eva-accession-pipeline/src/test/resources/properties/accession-pipeline-recover-state-test.properties b/eva-accession-pipeline/src/test/resources/properties/accession-pipeline-recover-state-test.properties index 2edb00288..ff59b51ba 100644 --- a/eva-accession-pipeline/src/test/resources/properties/accession-pipeline-recover-state-test.properties +++ b/eva-accession-pipeline/src/test/resources/properties/accession-pipeline-recover-state-test.properties @@ -4,14 +4,10 @@ accessioning.monotonic.test-pipeline-recover-state-ss.blockSize=30 accessioning.monotonic.test-pipeline-recover-state-ss.blockStartValue=5000000000 accessioning.monotonic.test-pipeline-recover-state-ss.nextBlockInterval=1000000000 -spring.datasource.driver-class-name=org.hsqldb.jdbcDriver -spring.datasource.url=jdbc:hsqldb:mem:db;sql.syntax_pgs=true;DB_CLOSE_DELAY=-1 -spring.datasource.username=SA -spring.datasource.password= -spring.sql.init.schema-locations=classpath:test-data/contiguous_id_blocks_schema.sql -spring.sql.init.data-locations=classpath:test-data/contiguous_id_blocks_recover_state_data.sql -spring.sql.init.mode=always -spring.jpa.hibernate.ddl-auto=none +continuous.id.blocks.datasource.driver-class-name=org.hsqldb.jdbc.JDBCDriver +continuous.id.blocks.datasource.jdbc-url=jdbc:hsqldb:mem:db;sql.syntax_pgs=true;DB_CLOSE_DELAY=-1 +continuous.id.blocks.datasource.username=SA +continuous.id.blocks.datasource.password= parameters.assemblyAccession=assembly parameters.taxonomyAccession=1111 diff --git a/eva-accession-pipeline/src/test/resources/properties/accession-pipeline-recover-test.properties b/eva-accession-pipeline/src/test/resources/properties/accession-pipeline-recover-test.properties index 26d8215a4..de830075f 100644 --- a/eva-accession-pipeline/src/test/resources/properties/accession-pipeline-recover-test.properties +++ b/eva-accession-pipeline/src/test/resources/properties/accession-pipeline-recover-test.properties @@ -8,14 +8,12 @@ eva.count-stats.url=http://localhost:8080 eva.count-stats.username=username eva.count-stats.password=password -spring.datasource.driver-class-name=org.hsqldb.jdbcDriver -spring.datasource.url=jdbc:hsqldb:mem:db;sql.syntax_pgs=true;DB_CLOSE_DELAY=-1 -spring.datasource.username=SA -spring.datasource.password= -spring.sql.init.schema-locations=classpath:test-data/contiguous_id_blocks_schema.sql +continuous.id.blocks.datasource.driver-class-name=org.hsqldb.jdbc.JDBCDriver +continuous.id.blocks.datasource.jdbc-url=jdbc:hsqldb:mem:db;sql.syntax_pgs=true;DB_CLOSE_DELAY=-1 +continuous.id.blocks.datasource.username=SA +continuous.id.blocks.datasource.password= + spring.sql.init.data-locations=classpath:test-data/contiguous_id_blocks_data.sql -spring.sql.init.mode=always -spring.jpa.hibernate.ddl-auto=none parameters.assemblyAccession=assembly parameters.taxonomyAccession=1111 diff --git a/eva-accession-pipeline/src/test/resources/properties/accession-pipeline-test.properties b/eva-accession-pipeline/src/test/resources/properties/accession-pipeline-test.properties index 5a20cf25b..a503929da 100644 --- a/eva-accession-pipeline/src/test/resources/properties/accession-pipeline-test.properties +++ b/eva-accession-pipeline/src/test/resources/properties/accession-pipeline-test.properties @@ -5,6 +5,11 @@ eva.count-stats.url=http://localhost:8080 eva.count-stats.username=username eva.count-stats.password=password +continuous.id.blocks.datasource.driver-class-name=org.hsqldb.jdbc.JDBCDriver +continuous.id.blocks.datasource.jdbc-url=jdbc:hsqldb:mem:db;sql.syntax_pgs=true;DB_CLOSE_DELAY=-1 +continuous.id.blocks.datasource.username=SA +continuous.id.blocks.datasource.password= + accessioning.monotonic.test-pipeline-ss.blockSize=100000 accessioning.monotonic.test-pipeline-ss.blockStartValue=5000000000 accessioning.monotonic.test-pipeline-ss.nextBlockInterval=1000000000 diff --git a/eva-accession-pipeline/src/test/resources/properties/duplicate-ss-acc-qc-test.properties b/eva-accession-pipeline/src/test/resources/properties/duplicate-ss-acc-qc-test.properties index 183cdbf90..571b8e234 100644 --- a/eva-accession-pipeline/src/test/resources/properties/duplicate-ss-acc-qc-test.properties +++ b/eva-accession-pipeline/src/test/resources/properties/duplicate-ss-acc-qc-test.properties @@ -5,6 +5,11 @@ eva.count-stats.url=http://localhost:8080 eva.count-stats.username=username eva.count-stats.password=password +continuous.id.blocks.datasource.driver-class-name=org.hsqldb.jdbc.JDBCDriver +continuous.id.blocks.datasource.jdbc-url=jdbc:hsqldb:mem:db;sql.syntax_pgs=true;DB_CLOSE_DELAY=-1 +continuous.id.blocks.datasource.username=SA +continuous.id.blocks.datasource.password= + accessioning.monotonic.test-pipeline-ss.blockSize=100000 accessioning.monotonic.test-pipeline-ss.blockStartValue=5000000000 accessioning.monotonic.test-pipeline-ss.nextBlockInterval=1000000000 diff --git a/eva-accession-pipeline/src/test/resources/properties/qc-accession-pipeline-test.properties b/eva-accession-pipeline/src/test/resources/properties/qc-accession-pipeline-test.properties index 28c3df942..398a465b7 100644 --- a/eva-accession-pipeline/src/test/resources/properties/qc-accession-pipeline-test.properties +++ b/eva-accession-pipeline/src/test/resources/properties/qc-accession-pipeline-test.properties @@ -8,6 +8,11 @@ accessioning.monotonic.test-pipeline-ss.blockSize=100000 accessioning.monotonic.test-pipeline-ss.blockStartValue=5000000000 accessioning.monotonic.test-pipeline-ss.nextBlockInterval=1000000000 +continuous.id.blocks.datasource.driver-class-name=org.hsqldb.jdbc.JDBCDriver +continuous.id.blocks.datasource.jdbc-url=jdbc:hsqldb:mem:db;sql.syntax_pgs=true;DB_CLOSE_DELAY=-1 +continuous.id.blocks.datasource.username=SA +continuous.id.blocks.datasource.password= + parameters.assemblyAccession=assembly parameters.taxonomyAccession=1111 parameters.projectAccession=project diff --git a/eva-accession-pipeline/src/test/resources/properties/ss-accession-recovery.properties b/eva-accession-pipeline/src/test/resources/properties/ss-accession-recovery.properties index 343cf5fde..c651270b7 100644 --- a/eva-accession-pipeline/src/test/resources/properties/ss-accession-recovery.properties +++ b/eva-accession-pipeline/src/test/resources/properties/ss-accession-recovery.properties @@ -1,13 +1,9 @@ recovery.cutoff.days=14 -spring.datasource.driver-class-name=org.hsqldb.jdbcDriver -spring.datasource.url=jdbc:hsqldb:mem:db;sql.syntax_pgs=true;DB_CLOSE_DELAY=-1 -spring.datasource.username=SA -spring.datasource.password= -spring.sql.init.schema-locations=classpath:test-data/contiguous_id_blocks_schema.sql -spring.sql.init.data-locations=classpath:test-data/ss_accession_recovery_test_data.sql -spring.sql.init.mode=always -spring.jpa.hibernate.ddl-auto=none +continuous.id.blocks.datasource.driver-class-name=org.hsqldb.jdbc.JDBCDriver +continuous.id.blocks.datasource.jdbc-url=jdbc:hsqldb:mem:db;sql.syntax_pgs=true;DB_CLOSE_DELAY=-1 +continuous.id.blocks.datasource.username=SA +continuous.id.blocks.datasource.password= accessioning.submitted.categoryId=test-pipeline-ss diff --git a/eva-accession-pipeline/src/test/resources/test-data/contiguous_id_blocks_schema.sql b/eva-accession-pipeline/src/test/resources/test-data/contiguous_id_blocks_schema.sql index 495f9a4fb..32dfe369d 100644 --- a/eva-accession-pipeline/src/test/resources/test-data/contiguous_id_blocks_schema.sql +++ b/eva-accession-pipeline/src/test/resources/test-data/contiguous_id_blocks_schema.sql @@ -1,4 +1,8 @@ -CREATE TABLE contiguous_id_blocks ( +CREATE SEQUENCE IF NOT EXISTS contiguous_id_blocks_seq + START WITH 1 + INCREMENT BY 50; + +CREATE TABLE IF NOT EXISTS contiguous_id_blocks ( id bigint not NULL, application_instance_id varchar(255) not NULL, category_id varchar(255) not NULL, diff --git a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/Application.java b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/Application.java index 7c80c6fe7..86f86cfc1 100644 --- a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/Application.java +++ b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/Application.java @@ -15,20 +15,25 @@ */ package uk.ac.ebi.eva.accession.release; -import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.autoconfigure.batch.BatchAutoConfiguration; +import org.springframework.boot.autoconfigure.data.mongo.MongoDataAutoConfiguration; import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration; import org.springframework.boot.autoconfigure.jdbc.DataSourceTransactionManagerAutoConfiguration; import org.springframework.boot.autoconfigure.orm.jpa.HibernateJpaAutoConfiguration; import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.context.annotation.Import; +import uk.ac.ebi.eva.accession.core.configuration.InMemoryBatchConfiguration; -@EnableBatchProcessing @SpringBootApplication(exclude = { + MongoDataAutoConfiguration.class, DataSourceAutoConfiguration.class, DataSourceTransactionManagerAutoConfiguration.class, - HibernateJpaAutoConfiguration.class + HibernateJpaAutoConfiguration.class, + BatchAutoConfiguration.class }) +@Import(InMemoryBatchConfiguration.class) public class Application { public static void main(String[] args) { diff --git a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/RunnerConfiguration.java b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/RunnerConfiguration.java index a0c05b893..1758f27fc 100644 --- a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/RunnerConfiguration.java +++ b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/RunnerConfiguration.java @@ -16,15 +16,40 @@ */ package uk.ac.ebi.eva.accession.release.configuration; +import org.springframework.batch.core.ExitStatus; +import org.springframework.batch.core.Job; +import org.springframework.batch.core.JobExecution; +import org.springframework.batch.core.JobParametersBuilder; +import org.springframework.batch.core.launch.JobLauncher; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.CommandLineRunner; +import org.springframework.context.ApplicationContext; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; +import uk.ac.ebi.eva.accession.core.configuration.nonhuman.MongoConfiguration; import uk.ac.ebi.eva.commons.batch.job.JobExecutionApplicationListener; @Configuration +@Import({MongoConfiguration.class}) public class RunnerConfiguration { @Bean public JobExecutionApplicationListener jobExecutionApplicationListener() { return new JobExecutionApplicationListener(); } + + @Bean + public CommandLineRunner runJob(JobLauncher jobLauncher, ApplicationContext context, + @Value("${spring.batch.job.names}") String jobName) { + return args -> { + Job job = context.getBean(jobName, Job.class); + JobExecution execution = jobLauncher.run(job, new JobParametersBuilder() + .toJobParameters()); + + if (!execution.getExitStatus().equals(ExitStatus.COMPLETED)) { + throw new RuntimeException("Job failed with status: " + execution.getExitStatus()); + } + }; + } } \ No newline at end of file diff --git a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/jobs/ActiveAccessionReleaseFromDBJobConfiguration.java b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/jobs/ActiveAccessionReleaseFromDBJobConfiguration.java index fe595823c..c53f497fd 100644 --- a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/jobs/ActiveAccessionReleaseFromDBJobConfiguration.java +++ b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/jobs/ActiveAccessionReleaseFromDBJobConfiguration.java @@ -2,9 +2,7 @@ import org.springframework.batch.core.Job; import org.springframework.batch.core.Step; -import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.batch.core.job.builder.JobBuilder; -import org.springframework.batch.core.launch.support.RunIdIncrementer; import org.springframework.batch.core.repository.JobRepository; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; @@ -15,7 +13,6 @@ import static uk.ac.ebi.eva.accession.release.configuration.BeanNames.ACTIVE_ACCESSIONS_RELEASE_FROM_DB_STEP; @Configuration -@EnableBatchProcessing public class ActiveAccessionReleaseFromDBJobConfiguration { @Autowired @@ -25,7 +22,6 @@ public class ActiveAccessionReleaseFromDBJobConfiguration { @Bean(ACTIVE_ACCESSIONS_RELEASE_FROM_DB_JOB) public Job activeAccessionReleaseFromDBJob(JobRepository jobRepository) { return new JobBuilder(ACTIVE_ACCESSIONS_RELEASE_FROM_DB_JOB, jobRepository) - .incrementer(new RunIdIncrementer()) .start(activeAccessionsReleaseFromDBStep) .build(); } diff --git a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/jobs/DumpRSAccessionsJobConfiguration.java b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/jobs/DumpRSAccessionsJobConfiguration.java index e8be4c938..a81c2c452 100644 --- a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/jobs/DumpRSAccessionsJobConfiguration.java +++ b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/jobs/DumpRSAccessionsJobConfiguration.java @@ -3,9 +3,7 @@ import org.springframework.batch.core.Job; import org.springframework.batch.core.Step; -import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.batch.core.job.builder.JobBuilder; -import org.springframework.batch.core.launch.support.RunIdIncrementer; import org.springframework.batch.core.repository.JobRepository; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; @@ -18,7 +16,6 @@ import static uk.ac.ebi.eva.accession.release.configuration.BeanNames.DUMP_MERGED_AND_DEPRECATED_ACCESSIONS_STEP; @Configuration -@EnableBatchProcessing public class DumpRSAccessionsJobConfiguration { @Autowired @@ -32,7 +29,6 @@ public class DumpRSAccessionsJobConfiguration { @Bean(DUMP_ACTIVE_ACCESSIONS_JOB) public Job dumpActiveAccessionJob(JobRepository jobRepository) { return new JobBuilder(DUMP_ACTIVE_ACCESSIONS_JOB, jobRepository) - .incrementer(new RunIdIncrementer()) .start(dumpActiveAccessionsStep) .build(); } @@ -40,7 +36,6 @@ public Job dumpActiveAccessionJob(JobRepository jobRepository) { @Bean(DUMP_MERGED_AND_DEPRECATED_ACCESSIONS_JOB) public Job dumpMergedAndDeprecatedAccessionJob(JobRepository jobRepository) { return new JobBuilder(DUMP_MERGED_AND_DEPRECATED_ACCESSIONS_JOB, jobRepository) - .incrementer(new RunIdIncrementer()) .start(dumpMergedAndDeprecatedAccessionsStep) .build(); } diff --git a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/jobs/MergedAndDeprecatedAccessionReleaseFromDBJobConfiguration.java b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/jobs/MergedAndDeprecatedAccessionReleaseFromDBJobConfiguration.java index acf2c36b6..d559d3abd 100644 --- a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/jobs/MergedAndDeprecatedAccessionReleaseFromDBJobConfiguration.java +++ b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/jobs/MergedAndDeprecatedAccessionReleaseFromDBJobConfiguration.java @@ -2,9 +2,7 @@ import org.springframework.batch.core.Job; import org.springframework.batch.core.Step; -import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.batch.core.job.builder.JobBuilder; -import org.springframework.batch.core.launch.support.RunIdIncrementer; import org.springframework.batch.core.repository.JobRepository; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; @@ -15,7 +13,6 @@ import static uk.ac.ebi.eva.accession.release.configuration.BeanNames.MERGED_AND_DEPRECATED_ACCESSIONS_RELEASE_FROM_DB_STEP; @Configuration -@EnableBatchProcessing public class MergedAndDeprecatedAccessionReleaseFromDBJobConfiguration { @Autowired @@ -25,7 +22,6 @@ public class MergedAndDeprecatedAccessionReleaseFromDBJobConfiguration { @Bean(MERGED_AND_DEPRECATED_ACCESSIONS_RELEASE_FROM_DB_JOB) public Job mergedAndDeprecatedAccessionReleaseFromDBJob(JobRepository jobRepository) { return new JobBuilder(MERGED_AND_DEPRECATED_ACCESSIONS_RELEASE_FROM_DB_JOB, jobRepository) - .incrementer(new RunIdIncrementer()) .start(mergedAndDeprecatedAccessionsReleaseFromDBStep) .build(); } diff --git a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/steps/ActiveAccessionReleaseFromDBStepConfiguration.java b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/steps/ActiveAccessionReleaseFromDBStepConfiguration.java index be3a1e5d5..7bbb10618 100644 --- a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/steps/ActiveAccessionReleaseFromDBStepConfiguration.java +++ b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/steps/ActiveAccessionReleaseFromDBStepConfiguration.java @@ -2,7 +2,6 @@ import htsjdk.variant.variantcontext.VariantContext; import org.springframework.batch.core.Step; -import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.batch.core.repository.JobRepository; import org.springframework.batch.core.step.builder.StepBuilder; import org.springframework.batch.core.step.tasklet.TaskletStep; @@ -17,13 +16,13 @@ import org.springframework.transaction.PlatformTransactionManager; import uk.ac.ebi.eva.commons.core.models.pipeline.Variant; +import static uk.ac.ebi.eva.accession.core.configuration.InMemoryBatchConfiguration.BATCH_TRANSACTION_MANAGER; import static uk.ac.ebi.eva.accession.release.configuration.BeanNames.ACTIVE_ACCESSIONS_RELEASE_FROM_DB_STEP; import static uk.ac.ebi.eva.accession.release.configuration.BeanNames.ACTIVE_ACCESSIONS_VARIANT_READER; import static uk.ac.ebi.eva.accession.release.configuration.BeanNames.EVA_RELEASE_WRITER; import static uk.ac.ebi.eva.accession.release.configuration.BeanNames.RELEASE_PROCESSOR; @Configuration -@EnableBatchProcessing public class ActiveAccessionReleaseFromDBStepConfiguration { @Autowired @@ -39,7 +38,9 @@ public class ActiveAccessionReleaseFromDBStepConfiguration { ItemStreamWriter accessionWriter; @Bean(ACTIVE_ACCESSIONS_RELEASE_FROM_DB_STEP) - public Step activeAccessionsReleaseFromDBStep(JobRepository jobRepository, PlatformTransactionManager transactionManager, + public Step activeAccessionsReleaseFromDBStep(JobRepository jobRepository, + @Qualifier(BATCH_TRANSACTION_MANAGER) + PlatformTransactionManager transactionManager, SimpleCompletionPolicy chunkSizeCompletionPolicy) { TaskletStep step = new StepBuilder(ACTIVE_ACCESSIONS_RELEASE_FROM_DB_STEP, jobRepository) .chunk(chunkSizeCompletionPolicy, transactionManager) diff --git a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/steps/DumpRSAccessionsStepConfiguration.java b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/steps/DumpRSAccessionsStepConfiguration.java index 762b22ac1..ed27062c8 100644 --- a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/steps/DumpRSAccessionsStepConfiguration.java +++ b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/steps/DumpRSAccessionsStepConfiguration.java @@ -1,7 +1,6 @@ package uk.ac.ebi.eva.accession.release.configuration.batch.steps; import org.springframework.batch.core.Step; -import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.batch.core.repository.JobRepository; import org.springframework.batch.core.step.builder.StepBuilder; import org.springframework.beans.factory.annotation.Autowired; @@ -12,12 +11,12 @@ import uk.ac.ebi.eva.accession.release.batch.io.DumpRSAccessionsInFile; import uk.ac.ebi.eva.accession.release.parameters.InputParameters; +import static uk.ac.ebi.eva.accession.core.configuration.InMemoryBatchConfiguration.BATCH_TRANSACTION_MANAGER; import static uk.ac.ebi.eva.accession.release.configuration.BeanNames.DUMP_ACTIVE_ACCESSIONS_STEP; import static uk.ac.ebi.eva.accession.release.configuration.BeanNames.DUMP_MERGED_AND_DEPRECATED_ACCESSIONS_STEP; import static uk.ac.ebi.eva.accession.release.configuration.BeanNames.DUMP_RS_ACCESSIONS_IN_FILE; @Configuration -@EnableBatchProcessing public class DumpRSAccessionsStepConfiguration { @Autowired @@ -25,7 +24,8 @@ public class DumpRSAccessionsStepConfiguration { private DumpRSAccessionsInFile dumpRSAccessionsInFile; @Bean(DUMP_ACTIVE_ACCESSIONS_STEP) - public Step dumpActiveAccessionStep(JobRepository jobRepository, PlatformTransactionManager transactionManager, + public Step dumpActiveAccessionStep(JobRepository jobRepository, + @Qualifier(BATCH_TRANSACTION_MANAGER) PlatformTransactionManager transactionManager, InputParameters inputParameters) { return new StepBuilder(DUMP_ACTIVE_ACCESSIONS_STEP, jobRepository) .tasklet((contribution, chunkContext) -> { @@ -37,7 +37,8 @@ public Step dumpActiveAccessionStep(JobRepository jobRepository, PlatformTransac } @Bean(DUMP_MERGED_AND_DEPRECATED_ACCESSIONS_STEP) - public Step dumpMergedAndDeprecatedAccessionStep(JobRepository jobRepository, PlatformTransactionManager transactionManager, + public Step dumpMergedAndDeprecatedAccessionStep(JobRepository jobRepository, + @Qualifier(BATCH_TRANSACTION_MANAGER) PlatformTransactionManager transactionManager, InputParameters inputParameters) { return new StepBuilder(DUMP_MERGED_AND_DEPRECATED_ACCESSIONS_STEP, jobRepository) .tasklet((contribution, chunkContext) -> { diff --git a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/steps/MergedAndDeprecatedAccessionReleaseFromDBStepConfiguration.java b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/steps/MergedAndDeprecatedAccessionReleaseFromDBStepConfiguration.java index a07221a2a..4dd0ec12d 100644 --- a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/steps/MergedAndDeprecatedAccessionReleaseFromDBStepConfiguration.java +++ b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/configuration/batch/steps/MergedAndDeprecatedAccessionReleaseFromDBStepConfiguration.java @@ -2,7 +2,6 @@ import htsjdk.variant.variantcontext.VariantContext; import org.springframework.batch.core.Step; -import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.batch.core.repository.JobRepository; import org.springframework.batch.core.step.builder.StepBuilder; import org.springframework.batch.core.step.tasklet.TaskletStep; @@ -17,13 +16,13 @@ import org.springframework.transaction.PlatformTransactionManager; import uk.ac.ebi.eva.commons.core.models.pipeline.Variant; +import static uk.ac.ebi.eva.accession.core.configuration.InMemoryBatchConfiguration.BATCH_TRANSACTION_MANAGER; import static uk.ac.ebi.eva.accession.release.configuration.BeanNames.EVA_MERGED_RELEASE_WRITER; import static uk.ac.ebi.eva.accession.release.configuration.BeanNames.MERGED_AND_DEPRECATED_ACCESSIONS_RELEASE_FROM_DB_STEP; import static uk.ac.ebi.eva.accession.release.configuration.BeanNames.MERGED_AND_DEPRECATED_ACCESSIONS_VARIANT_READER; import static uk.ac.ebi.eva.accession.release.configuration.BeanNames.RELEASE_PROCESSOR; @Configuration -@EnableBatchProcessing public class MergedAndDeprecatedAccessionReleaseFromDBStepConfiguration { @Autowired @@ -39,7 +38,9 @@ public class MergedAndDeprecatedAccessionReleaseFromDBStepConfiguration { ItemStreamWriter accessionWriter; @Bean(MERGED_AND_DEPRECATED_ACCESSIONS_RELEASE_FROM_DB_STEP) - public Step mergedAndDeprecatedAccessionsReleaseFromDBStep(JobRepository jobRepository, PlatformTransactionManager transactionManager, + public Step mergedAndDeprecatedAccessionsReleaseFromDBStep(JobRepository jobRepository, + @Qualifier(BATCH_TRANSACTION_MANAGER) + PlatformTransactionManager transactionManager, SimpleCompletionPolicy chunkSizeCompletionPolicy) { TaskletStep step = new StepBuilder(MERGED_AND_DEPRECATED_ACCESSIONS_RELEASE_FROM_DB_STEP, jobRepository) .chunk(chunkSizeCompletionPolicy, transactionManager) diff --git a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/parameters/InputParameters.java b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/parameters/InputParameters.java index 832e6fdfe..d17f6febc 100644 --- a/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/parameters/InputParameters.java +++ b/eva-accession-release/src/main/java/uk/ac/ebi/eva/accession/release/parameters/InputParameters.java @@ -33,8 +33,6 @@ public class InputParameters { private String outputFolder; - private boolean forceRestart; - private int chunkSize; private String accessionedVcf; @@ -97,14 +95,6 @@ public void setOutputFolder(String outputFolder) { this.outputFolder = outputFolder; } - public boolean isForceRestart() { - return forceRestart; - } - - public void setForceRestart(boolean forceRestart) { - this.forceRestart = forceRestart; - } - public int getChunkSize() { return chunkSize; } diff --git a/eva-accession-release/src/main/resources/application.properties b/eva-accession-release/src/main/resources/application.properties index 65cf102b2..0b1c7a873 100644 --- a/eva-accession-release/src/main/resources/application.properties +++ b/eva-accession-release/src/main/resources/application.properties @@ -5,7 +5,6 @@ parameters.taxonomyAccession= parameters.fasta= parameters.assemblyReportUrl= parameters.outputFolder= -parameters.forceRestart=false parameters.chunkSize=100 parameters.contigNaming=SEQUENCE_NAME diff --git a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/active/AccessionedVariantMongoReaderTest.java b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/active/AccessionedVariantMongoReaderTest.java index 14bb1febe..6a0278069 100644 --- a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/active/AccessionedVariantMongoReaderTest.java +++ b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/active/AccessionedVariantMongoReaderTest.java @@ -33,10 +33,10 @@ import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.MongoConfiguration; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.core.utils.MongoTestDataLoader; import uk.ac.ebi.eva.accession.release.collectionNames.DbsnpCollectionNames; -import uk.ac.ebi.eva.accession.release.test.configuration.MongoTestConfiguration; import uk.ac.ebi.eva.commons.core.models.pipeline.Variant; import java.util.ArrayList; diff --git a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/contig/ContigMongoReaderTest.java b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/contig/ContigMongoReaderTest.java index 17056b906..7df1ce539 100644 --- a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/contig/ContigMongoReaderTest.java +++ b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/contig/ContigMongoReaderTest.java @@ -29,11 +29,11 @@ import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.MongoConfiguration; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.core.utils.MongoTestDataLoader; import uk.ac.ebi.eva.accession.release.collectionNames.DbsnpCollectionNames; import uk.ac.ebi.eva.accession.release.collectionNames.EvaCollectionNames; -import uk.ac.ebi.eva.accession.release.test.configuration.MongoTestConfiguration; import java.util.ArrayList; import java.util.Arrays; diff --git a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/deprecated/DeprecatedVariantMongoReaderTest.java b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/deprecated/DeprecatedVariantMongoReaderTest.java index 75adc3d34..f47e90bec 100644 --- a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/deprecated/DeprecatedVariantMongoReaderTest.java +++ b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/deprecated/DeprecatedVariantMongoReaderTest.java @@ -40,9 +40,9 @@ import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpSubmittedVariantOperationEntity; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; import uk.ac.ebi.eva.accession.core.summary.ClusteredVariantSummaryFunction; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.release.collectionNames.DbsnpCollectionNames; -import uk.ac.ebi.eva.accession.release.test.configuration.MongoTestConfiguration; import uk.ac.ebi.eva.commons.core.models.pipeline.Variant; import java.util.ArrayList; diff --git a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/merged/MergedVariantMongoReaderTest.java b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/merged/MergedVariantMongoReaderTest.java index 257d5bee5..aecdaef2e 100644 --- a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/merged/MergedVariantMongoReaderTest.java +++ b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/merged/MergedVariantMongoReaderTest.java @@ -43,11 +43,11 @@ import uk.ac.ebi.eva.accession.core.model.eva.ClusteredVariantEntity; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; import uk.ac.ebi.eva.accession.core.summary.ClusteredVariantSummaryFunction; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.core.utils.MongoTestDataLoader; import uk.ac.ebi.eva.accession.release.collectionNames.DbsnpCollectionNames; import uk.ac.ebi.eva.accession.release.collectionNames.EvaCollectionNames; -import uk.ac.ebi.eva.accession.release.test.configuration.MongoTestConfiguration; import uk.ac.ebi.eva.commons.core.models.pipeline.Variant; import java.util.ArrayList; diff --git a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/merged_deprecated/MergedDeprecatedVariantMongoReaderTest.java b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/merged_deprecated/MergedDeprecatedVariantMongoReaderTest.java index ef211280c..20803bc22 100644 --- a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/merged_deprecated/MergedDeprecatedVariantMongoReaderTest.java +++ b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/batch/io/merged_deprecated/MergedDeprecatedVariantMongoReaderTest.java @@ -31,10 +31,10 @@ import uk.ac.ebi.ampt2d.commons.accession.core.models.EventType; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.MongoConfiguration; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpClusteredVariantOperationEntity; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.core.utils.MongoTestDataLoader; import uk.ac.ebi.eva.accession.release.collectionNames.DbsnpCollectionNames; -import uk.ac.ebi.eva.accession.release.test.configuration.MongoTestConfiguration; import java.util.ArrayList; import java.util.List; diff --git a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/configuration/batch/steps/DumpRSAccessionsTest.java b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/configuration/batch/steps/DumpRSAccessionsTest.java index 5308c0b55..eb65f6f96 100644 --- a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/configuration/batch/steps/DumpRSAccessionsTest.java +++ b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/configuration/batch/steps/DumpRSAccessionsTest.java @@ -30,15 +30,15 @@ import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.ampt2d.commons.accession.core.models.EventType; +import uk.ac.ebi.eva.accession.core.configuration.InMemoryBatchConfiguration; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpClusteredVariantEntity; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpClusteredVariantInactiveEntity; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpClusteredVariantOperationEntity; import uk.ac.ebi.eva.accession.core.model.eva.ClusteredVariantEntity; import uk.ac.ebi.eva.accession.core.model.eva.ClusteredVariantInactiveEntity; import uk.ac.ebi.eva.accession.core.model.eva.ClusteredVariantOperationEntity; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; -import uk.ac.ebi.eva.accession.release.test.configuration.BatchJobRepositoryTestConfiguration; -import uk.ac.ebi.eva.accession.release.test.configuration.MongoTestConfiguration; import uk.ac.ebi.eva.accession.release.test.configuration.ReleaseFromDBTestConfiguration; import uk.ac.ebi.eva.commons.core.models.VariantType; @@ -58,8 +58,8 @@ import static uk.ac.ebi.eva.accession.release.test.configuration.ReleaseFromDBTestConfiguration.TEST_DUMP_MERGED_AND_DEPRECATED_ACCESSIONS_JOB; @ExtendWith(SpringExtension.class) -@ContextConfiguration(classes = {MongoTestConfiguration.class, ReleaseFromDBTestConfiguration.class, - BatchJobRepositoryTestConfiguration.class}) +@ContextConfiguration(classes = {InMemoryBatchConfiguration.class, MongoTestConfiguration.class, + ReleaseFromDBTestConfiguration.class}) @TestPropertySource("classpath:dump-rs-accession-test.properties") public class DumpRSAccessionsTest extends MongoTestContainerHelper { private static final String accDumpFile = "src/test/resources/accDumpFile.csv"; diff --git a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/configuration/batch/steps/RSAccessionsReleaseFromDBTest.java b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/configuration/batch/steps/RSAccessionsReleaseFromDBTest.java index 527f11039..737fc9ca0 100644 --- a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/configuration/batch/steps/RSAccessionsReleaseFromDBTest.java +++ b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/configuration/batch/steps/RSAccessionsReleaseFromDBTest.java @@ -30,6 +30,7 @@ import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.ampt2d.commons.accession.core.models.EventType; +import uk.ac.ebi.eva.accession.core.configuration.InMemoryBatchConfiguration; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpClusteredVariantEntity; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpSubmittedVariantEntity; import uk.ac.ebi.eva.accession.core.model.eva.ClusteredVariantEntity; @@ -38,11 +39,10 @@ import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantInactiveEntity; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantOperationEntity; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.release.parameters.InputParameters; import uk.ac.ebi.eva.accession.release.parameters.ReportPathResolver; -import uk.ac.ebi.eva.accession.release.test.configuration.BatchJobRepositoryTestConfiguration; -import uk.ac.ebi.eva.accession.release.test.configuration.MongoTestConfiguration; import uk.ac.ebi.eva.accession.release.test.configuration.ReleaseFromDBTestConfiguration; import uk.ac.ebi.eva.commons.core.models.VariantType; import uk.ac.ebi.eva.commons.core.utils.FileUtils; @@ -71,8 +71,8 @@ import static uk.ac.ebi.eva.accession.release.test.configuration.ReleaseFromDBTestConfiguration.TEST_RELEASE_MERGED_AND_DEPRECATED_ACCESSIONS_JOB; @ExtendWith(SpringExtension.class) -@ContextConfiguration(classes = {MongoTestConfiguration.class, ReleaseFromDBTestConfiguration.class, - BatchJobRepositoryTestConfiguration.class}) +@ContextConfiguration(classes = {InMemoryBatchConfiguration.class, MongoTestConfiguration.class, + ReleaseFromDBTestConfiguration.class}) @TestPropertySource("classpath:dump-rs-accession-test.properties") public class RSAccessionsReleaseFromDBTest extends MongoTestContainerHelper { private static final String testRunDir = "src/test/resources/release-test-run"; diff --git a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/test/configuration/BatchJobRepositoryTestConfiguration.java b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/test/configuration/BatchJobRepositoryTestConfiguration.java deleted file mode 100644 index a4ef0782d..000000000 --- a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/test/configuration/BatchJobRepositoryTestConfiguration.java +++ /dev/null @@ -1,22 +0,0 @@ -package uk.ac.ebi.eva.accession.release.test.configuration; - -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Primary; -import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseBuilder; -import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseType; - -import javax.sql.DataSource; - -@Configuration -public class BatchJobRepositoryTestConfiguration { - - @Bean - @Primary - public DataSource dataSource() { - return new EmbeddedDatabaseBuilder() - .setType(EmbeddedDatabaseType.HSQL) - .addScript("org/springframework/batch/core/schema-hsqldb.sql") - .build(); - } -} \ No newline at end of file diff --git a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/test/configuration/BatchTestConfiguration.java b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/test/configuration/BatchTestConfiguration.java index c90817cb3..3181faf7b 100644 --- a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/test/configuration/BatchTestConfiguration.java +++ b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/test/configuration/BatchTestConfiguration.java @@ -17,7 +17,6 @@ package uk.ac.ebi.eva.accession.release.test.configuration; import org.springframework.batch.test.JobLauncherTestUtils; -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Import; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.MongoConfiguration; @@ -26,7 +25,6 @@ import uk.ac.ebi.eva.accession.release.configuration.batch.policies.IllegalStartSkipPolicyConfiguration; import uk.ac.ebi.eva.commons.batch.job.JobExecutionApplicationListener; -@EnableAutoConfiguration @Import({MongoConfiguration.class, InputParametersConfiguration.class, ChunkSizeCompletionPolicyConfiguration.class, diff --git a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/test/configuration/MongoTestConfiguration.java b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/test/configuration/MongoTestConfiguration.java deleted file mode 100644 index 3051afd22..000000000 --- a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/test/configuration/MongoTestConfiguration.java +++ /dev/null @@ -1,105 +0,0 @@ -/* - * - * Copyright 2018 EMBL - European Bioinformatics Institute - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -package uk.ac.ebi.eva.accession.release.test.configuration; - -import com.mongodb.ReadPreference; -import com.mongodb.WriteConcern; -import com.mongodb.client.MongoClient; -import com.mongodb.client.MongoClients; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.domain.EntityScan; -import org.springframework.boot.test.autoconfigure.data.mongo.AutoConfigureDataMongo; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Primary; -import org.springframework.data.mongodb.MongoDatabaseFactory; -import org.springframework.data.mongodb.config.EnableMongoAuditing; -import org.springframework.data.mongodb.core.MongoTemplate; -import org.springframework.data.mongodb.core.SimpleMongoClientDatabaseFactory; -import org.springframework.data.mongodb.core.WriteResultChecking; -import org.springframework.data.mongodb.core.convert.DefaultDbRefResolver; -import org.springframework.data.mongodb.core.convert.DefaultMongoTypeMapper; -import org.springframework.data.mongodb.core.convert.MappingMongoConverter; -import org.springframework.data.mongodb.core.convert.MongoCustomConversions; -import org.springframework.data.mongodb.core.mapping.MongoMappingContext; -import org.springframework.data.mongodb.repository.config.EnableMongoRepositories; - -import java.util.Collections; - -@Configuration -@EntityScan(basePackages = {"uk.ac.ebi.eva.accession.core.persistence"}) -@EnableMongoRepositories(basePackages = { - "uk.ac.ebi.eva.accession.core.persistence", - "uk.ac.ebi.ampt2d.commons.accession.persistence.mongodb.repository"}) -@EnableMongoAuditing -@AutoConfigureDataMongo -public class MongoTestConfiguration { - - @Value("${spring.data.mongodb.uri}") - private String mongoUri; - - @Value("${spring.data.mongodb.database}") - private String database; - - @Value("${mongodb.read-preference:primary}") - private String readPreference; - - @Bean - @Primary - public MongoClient mongoClient() { - return MongoClients.create(mongoUri); - } - - @Bean("primaryFactory") - @Primary - public MongoDatabaseFactory mongoDbFactory() { - return new SimpleMongoClientDatabaseFactory(mongoClient(), database); - } - - @Bean - @Primary - public MongoCustomConversions mongoCustomConversions() { - return new MongoCustomConversions(Collections.emptyList()); - } - - @Bean - @Primary - public MongoMappingContext mongoMappingContext(MongoCustomConversions conversions) { - MongoMappingContext context = new MongoMappingContext(); - context.setSimpleTypeHolder(conversions.getSimpleTypeHolder()); - context.setAutoIndexCreation(false); - return context; - } - - @Bean - @Primary - public MongoTemplate mongoTemplate(MongoMappingContext mongoMappingContext, - MongoCustomConversions conversions) { - MappingMongoConverter converter = new MappingMongoConverter( - new DefaultDbRefResolver(mongoDbFactory()), mongoMappingContext); - converter.setCustomConversions(conversions); - converter.setTypeMapper(new DefaultMongoTypeMapper(null)); - converter.afterPropertiesSet(); - MongoTemplate mongoTemplate = new MongoTemplate(mongoDbFactory(), converter); - mongoTemplate.setReadPreference(ReadPreference.valueOf(readPreference)); - mongoTemplate.setWriteConcern(WriteConcern.MAJORITY); - mongoTemplate.setWriteResultChecking(WriteResultChecking.EXCEPTION); - return mongoTemplate; - } - -} diff --git a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/test/configuration/ReleaseFromDBTestConfiguration.java b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/test/configuration/ReleaseFromDBTestConfiguration.java index 8f84a0cf6..ca43e079d 100644 --- a/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/test/configuration/ReleaseFromDBTestConfiguration.java +++ b/eva-accession-release/src/test/java/uk/ac/ebi/eva/accession/release/test/configuration/ReleaseFromDBTestConfiguration.java @@ -5,7 +5,6 @@ import org.springframework.batch.core.repository.JobRepository; import org.springframework.batch.test.JobLauncherTestUtils; import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Import; import uk.ac.ebi.eva.accession.release.configuration.InputParametersConfiguration; @@ -29,8 +28,6 @@ import static uk.ac.ebi.eva.accession.release.configuration.BeanNames.DUMP_MERGED_AND_DEPRECATED_ACCESSIONS_JOB; import static uk.ac.ebi.eva.accession.release.configuration.BeanNames.MERGED_AND_DEPRECATED_ACCESSIONS_RELEASE_FROM_DB_JOB; - -@EnableAutoConfiguration @Import({ChunkSizeCompletionPolicyConfiguration.class, IllegalStartSkipPolicyConfiguration.class, InputParametersConfiguration.class, diff --git a/eva-accession-release/src/test/resources/application.properties b/eva-accession-release/src/test/resources/application.properties index b8fb0bd6e..1aff6f39c 100644 --- a/eva-accession-release/src/test/resources/application.properties +++ b/eva-accession-release/src/test/resources/application.properties @@ -12,7 +12,6 @@ parameters.taxonomyAccession=60711 parameters.fasta=src/test/resources/input-files/fasta/GCF_000409795.2.test.fna parameters.assemblyReportUrl=file:src/test/resources/input-files/assembly-report/GCF_000409795.2_Chlorocebus_sabeus_1.1_assembly_report.txt parameters.outputFolder=/tmp/ -parameters.forceRestart=false parameters.chunkSize=1000 parameters.contigNaming=SEQUENCE_NAME diff --git a/eva-accession-release/src/test/resources/dump-rs-accession-test.properties b/eva-accession-release/src/test/resources/dump-rs-accession-test.properties index ee7516341..beec7f25e 100644 --- a/eva-accession-release/src/test/resources/dump-rs-accession-test.properties +++ b/eva-accession-release/src/test/resources/dump-rs-accession-test.properties @@ -12,7 +12,6 @@ parameters.taxonomyAccession=60711 parameters.fasta=src/test/resources/input-files/fasta/GCF_000409795.2.test.fna parameters.assemblyReportUrl=file:src/test/resources/input-files/assembly-report/GCF_000409795.2_Chlorocebus_sabeus_1.1_assembly_report.txt parameters.outputFolder=src/test/resources/release-test-run -parameters.forceRestart=false parameters.chunkSize=5 parameters.contigNaming=SEQUENCE_NAME diff --git a/eva-accession-release/src/test/resources/incremental-release-pipeline-test.properties b/eva-accession-release/src/test/resources/incremental-release-pipeline-test.properties deleted file mode 100644 index 70140cc36..000000000 --- a/eva-accession-release/src/test/resources/incremental-release-pipeline-test.properties +++ /dev/null @@ -1,27 +0,0 @@ -accessioning.instanceId=test-instance-01 -accessioning.submitted.categoryId=test-release-ss - -accessioning.monotonic.test-pipeline-ss.blockSize=100000 -accessioning.monotonic.test-pipeline-ss.blockStartValue=5000000000 -accessioning.monotonic.test-pipeline-ss.nextBlockInterval=1000000000 - -spring.jpa.show-sql=true - -parameters.assemblyAccession=GCA_000409795.2 -parameters.fasta=src/test/resources/input-files/fasta/GCF_000409795.2.test.fna -parameters.accessionedVcf=src/test/resources/test-data/incremental_release/study1.accessioned.vcf -parameters.assemblyReportUrl=file:src/test/resources/input-files/assembly-report/GCF_000409795.2_Chlorocebus_sabeus_1.1_assembly_report.txt -parameters.outputFolder=/tmp/ -parameters.forceRestart=false -parameters.chunkSize=5 -parameters.contigNaming=INSDC - -spring.data.mongodb.database=test-db -mongodb.read-preference=primary - -# See https://github.com/spring-projects/spring-boot/wiki/Spring-Boot-2.1-Release-Notes#bean-overriding -spring.main.allow-bean-definition-overriding=true - -# to fix exception javax.management.InstanceAlreadyExistsException: com.zaxxer.hikari:name=dataSource,type=HikariDataSource -# see https://stackoverflow.com/a/51798043/2375586 -spring.jmx.enabled=false diff --git a/eva-accession-release/src/test/resources/release-pipeline-test.properties b/eva-accession-release/src/test/resources/release-pipeline-test.properties deleted file mode 100644 index 48df3dc06..000000000 --- a/eva-accession-release/src/test/resources/release-pipeline-test.properties +++ /dev/null @@ -1,28 +0,0 @@ -accessioning.instanceId=test-instance-01 -accessioning.submitted.categoryId=test-release-ss - -accessioning.monotonic.test-pipeline-ss.blockSize=100000 -accessioning.monotonic.test-pipeline-ss.blockStartValue=5000000000 -accessioning.monotonic.test-pipeline-ss.nextBlockInterval=1000000000 - -spring.jpa.show-sql=true - -parameters.assemblyAccession=GCA_000409795.2 -parameters.taxonomyAccession=60711 -parameters.fasta=src/test/resources/input-files/fasta/GCF_000409795.2.test.fna -parameters.assemblyReportUrl=file:src/test/resources/input-files/assembly-report/GCF_000409795.2_Chlorocebus_sabeus_1.1_assembly_report.txt -parameters.outputFolder=/tmp/ -parameters.forceRestart=false -parameters.chunkSize=5 -parameters.contigNaming=SEQUENCE_NAME - - -spring.data.mongodb.database=test-db -mongodb.read-preference=primary - -# See https://github.com/spring-projects/spring-boot/wiki/Spring-Boot-2.1-Release-Notes#bean-overriding -spring.main.allow-bean-definition-overriding=true - -# to fix exception javax.management.InstanceAlreadyExistsException: com.zaxxer.hikari:name=dataSource,type=HikariDataSource -# see https://stackoverflow.com/a/51798043/2375586 -spring.jmx.enabled=false diff --git a/eva-accession-ws/pom.xml b/eva-accession-ws/pom.xml index a9e697f1c..897fdfe92 100644 --- a/eva-accession-ws/pom.xml +++ b/eva-accession-ws/pom.xml @@ -42,6 +42,11 @@ springdoc-openapi-starter-webmvc-ui + + org.postgresql + postgresql + + uk.ac.ebi.eva eva-accession-core @@ -53,6 +58,7 @@ org.hsqldb hsqldb + test diff --git a/eva-accession-ws/src/main/java/uk/ac/ebi/eva/accession/ws/EvaAccessionApplication.java b/eva-accession-ws/src/main/java/uk/ac/ebi/eva/accession/ws/EvaAccessionApplication.java index a4f3e3c30..100713ee1 100644 --- a/eva-accession-ws/src/main/java/uk/ac/ebi/eva/accession/ws/EvaAccessionApplication.java +++ b/eva-accession-ws/src/main/java/uk/ac/ebi/eva/accession/ws/EvaAccessionApplication.java @@ -19,11 +19,25 @@ import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.autoconfigure.batch.BatchAutoConfiguration; import org.springframework.boot.autoconfigure.data.mongo.MongoDataAutoConfiguration; +import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration; +import org.springframework.boot.autoconfigure.jdbc.DataSourceTransactionManagerAutoConfiguration; import org.springframework.boot.autoconfigure.mongo.MongoAutoConfiguration; +import org.springframework.boot.autoconfigure.orm.jpa.HibernateJpaAutoConfiguration; import org.springframework.boot.web.servlet.support.SpringBootServletInitializer; +import org.springframework.context.annotation.Import; +import uk.ac.ebi.eva.accession.core.configuration.ContiguousIdBlocksDataSourceConfiguration; -@SpringBootApplication(exclude = {MongoAutoConfiguration.class, MongoDataAutoConfiguration.class}) +@SpringBootApplication(exclude = { + MongoAutoConfiguration.class, + MongoDataAutoConfiguration.class, + DataSourceTransactionManagerAutoConfiguration.class, + DataSourceAutoConfiguration.class, + HibernateJpaAutoConfiguration.class, + BatchAutoConfiguration.class +}) +@Import({ContiguousIdBlocksDataSourceConfiguration.class}) public class EvaAccessionApplication extends SpringBootServletInitializer { public static void main(String[] args) { diff --git a/eva-accession-ws/src/main/java/uk/ac/ebi/eva/accession/ws/rest/EvaControllerAdvice.java b/eva-accession-ws/src/main/java/uk/ac/ebi/eva/accession/ws/rest/EvaControllerAdvice.java index d1c79691e..5aa1839ac 100644 --- a/eva-accession-ws/src/main/java/uk/ac/ebi/eva/accession/ws/rest/EvaControllerAdvice.java +++ b/eva-accession-ws/src/main/java/uk/ac/ebi/eva/accession/ws/rest/EvaControllerAdvice.java @@ -15,6 +15,7 @@ */ package uk.ac.ebi.eva.accession.ws.rest; +import jakarta.servlet.http.HttpServletRequest; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Value; @@ -29,7 +30,6 @@ import uk.ac.ebi.ampt2d.commons.accession.rest.BasicRestControllerAdvice; import uk.ac.ebi.ampt2d.commons.accession.rest.dto.ErrorMessage; -import jakarta.servlet.http.HttpServletRequest; import java.net.URI; /** diff --git a/eva-accession-ws/src/main/resources/application.properties b/eva-accession-ws/src/main/resources/application.properties index f524486de..cdb4e4b78 100644 --- a/eva-accession-ws/src/main/resources/application.properties +++ b/eva-accession-ws/src/main/resources/application.properties @@ -34,3 +34,8 @@ spring.main.allow-bean-definition-overriding=true spring.jmx.default-domain=eva.ebi.ac.uk.|timestamp| management.metrics.binders.jvm.enabled=false + +continuous.id.blocks.datasource.driver-class-name=org.postgresql.Driver +continuous.id.blocks.datasource.jdbc-url=|eva.accession.jdbc.ur| +continuous.id.blocks.datasource.username=|eva.accession.user| +continuous.id.blocks.datasource.password=|eva.accession.password| \ No newline at end of file diff --git a/eva-accession-ws/src/test/java/uk/ac/ebi/eva/accession/ws/ClusteredVariantHistoryEndPointTest.java b/eva-accession-ws/src/test/java/uk/ac/ebi/eva/accession/ws/ClusteredVariantHistoryEndPointTest.java index 9a716ab72..a3880aa25 100644 --- a/eva-accession-ws/src/test/java/uk/ac/ebi/eva/accession/ws/ClusteredVariantHistoryEndPointTest.java +++ b/eva-accession-ws/src/test/java/uk/ac/ebi/eva/accession/ws/ClusteredVariantHistoryEndPointTest.java @@ -37,13 +37,14 @@ import uk.ac.ebi.ampt2d.commons.accession.core.models.EventType; import uk.ac.ebi.ampt2d.commons.accession.rest.dto.AccessionResponseDTO; import uk.ac.ebi.ampt2d.commons.accession.rest.dto.HistoryEventDTO; +import uk.ac.ebi.eva.accession.core.configuration.ContiguousIdBlocksDataSourceConfiguration; import uk.ac.ebi.eva.accession.core.model.ClusteredVariant; import uk.ac.ebi.eva.accession.core.model.IClusteredVariant; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.core.utils.MongoTestDataLoader; import uk.ac.ebi.eva.accession.ws.dto.VariantHistory; import uk.ac.ebi.eva.accession.ws.rest.ClusteredVariantsRestController; -import uk.ac.ebi.eva.accession.ws.test.MongoTestConfiguration; import uk.ac.ebi.eva.commons.core.models.contigalias.ContigNamingConvention; import java.util.Arrays; @@ -54,7 +55,7 @@ @ExtendWith(SpringExtension.class) @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) -@Import({MongoTestConfiguration.class}) +@Import({MongoTestConfiguration.class, ContiguousIdBlocksDataSourceConfiguration.class}) @TestPropertySource("classpath:accession-ws-test.properties") public class ClusteredVariantHistoryEndPointTest extends MongoTestContainerHelper { diff --git a/eva-accession-ws/src/test/java/uk/ac/ebi/eva/accession/ws/ClusteredVariantsRestControllerTest.java b/eva-accession-ws/src/test/java/uk/ac/ebi/eva/accession/ws/ClusteredVariantsRestControllerTest.java index 3718a92b6..636c05d0e 100644 --- a/eva-accession-ws/src/test/java/uk/ac/ebi/eva/accession/ws/ClusteredVariantsRestControllerTest.java +++ b/eva-accession-ws/src/test/java/uk/ac/ebi/eva/accession/ws/ClusteredVariantsRestControllerTest.java @@ -49,6 +49,7 @@ import uk.ac.ebi.ampt2d.commons.accession.hashing.SHA1HashingFunction; import uk.ac.ebi.ampt2d.commons.accession.persistence.mongodb.document.AccessionedDocument; import uk.ac.ebi.ampt2d.commons.accession.rest.dto.AccessionResponseDTO; +import uk.ac.ebi.eva.accession.core.configuration.ContiguousIdBlocksDataSourceConfiguration; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.ClusteredVariantAccessioningConfiguration; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.SubmittedVariantAccessioningConfiguration; import uk.ac.ebi.eva.accession.core.contigalias.ContigAliasService; @@ -72,10 +73,10 @@ import uk.ac.ebi.eva.accession.core.service.nonhuman.dbsnp.DbsnpClusteredVariantMonotonicAccessioningService; import uk.ac.ebi.eva.accession.core.summary.ClusteredVariantSummaryFunction; import uk.ac.ebi.eva.accession.core.summary.SubmittedVariantSummaryFunction; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.ws.rest.ClusteredVariantsRestController; import uk.ac.ebi.eva.accession.ws.service.ClusteredVariantsBeaconService; -import uk.ac.ebi.eva.accession.ws.test.MongoTestConfiguration; import uk.ac.ebi.eva.accession.ws.test.NoContigTranslationArgumentMatcher; import uk.ac.ebi.eva.commons.beacon.models.BeaconAlleleRequest; import uk.ac.ebi.eva.commons.beacon.models.BeaconAlleleResponse; @@ -112,7 +113,7 @@ @ExtendWith(SpringExtension.class) @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) @Import({ClusteredVariantAccessioningConfiguration.class, SubmittedVariantAccessioningConfiguration.class, - MongoTestConfiguration.class}) + MongoTestConfiguration.class, ContiguousIdBlocksDataSourceConfiguration.class}) @TestPropertySource("classpath:accession-ws-test.properties") public class ClusteredVariantsRestControllerTest extends MongoTestContainerHelper { diff --git a/eva-accession-ws/src/test/java/uk/ac/ebi/eva/accession/ws/SubmittedVariantsRestControllerTest.java b/eva-accession-ws/src/test/java/uk/ac/ebi/eva/accession/ws/SubmittedVariantsRestControllerTest.java index eda734d86..1ee3a2307 100644 --- a/eva-accession-ws/src/test/java/uk/ac/ebi/eva/accession/ws/SubmittedVariantsRestControllerTest.java +++ b/eva-accession-ws/src/test/java/uk/ac/ebi/eva/accession/ws/SubmittedVariantsRestControllerTest.java @@ -30,12 +30,15 @@ import org.springframework.boot.test.web.client.TestRestTemplate; import org.springframework.context.annotation.Import; import org.springframework.core.ParameterizedTypeReference; +import org.springframework.core.io.ClassPathResource; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.http.HttpHeaders; import org.springframework.http.HttpMethod; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; +import org.springframework.jdbc.datasource.init.ResourceDatabasePopulator; import org.springframework.mock.web.MockHttpServletResponse; +import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit.jupiter.SpringExtension; import org.springframework.web.server.ResponseStatusException; @@ -47,6 +50,7 @@ import uk.ac.ebi.ampt2d.commons.accession.core.models.GetOrCreateAccessionWrapper; import uk.ac.ebi.ampt2d.commons.accession.rest.controllers.BasicRestController; import uk.ac.ebi.ampt2d.commons.accession.rest.dto.AccessionResponseDTO; +import uk.ac.ebi.eva.accession.core.configuration.ContiguousIdBlocksDataSourceConfiguration; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.SubmittedVariantAccessioningConfiguration; import uk.ac.ebi.eva.accession.core.contigalias.ContigAliasService; import uk.ac.ebi.eva.accession.core.model.ISubmittedVariant; @@ -67,6 +71,10 @@ import uk.ac.ebi.eva.accession.ws.test.NoContigTranslationArgumentMatcher; import uk.ac.ebi.eva.commons.core.models.contigalias.ContigNamingConvention; +import javax.sql.DataSource; +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -90,9 +98,10 @@ @ExtendWith(SpringExtension.class) @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) -@Import({SubmittedVariantAccessioningConfiguration.class}) +@Import({SubmittedVariantAccessioningConfiguration.class, ContiguousIdBlocksDataSourceConfiguration.class}) @TestPropertySource("classpath:accession-ws-test.properties") -public class SubmittedVariantsRestControllerTest extends MongoTestContainerHelper{ +@DirtiesContext(classMode = DirtiesContext.ClassMode.BEFORE_EACH_TEST_METHOD) +public class SubmittedVariantsRestControllerTest extends MongoTestContainerHelper { private static String TEST_APPLICATION_INSTANCE_ID = "test-application-instance-id"; private static final String URL = "/v1/submitted-variants/"; @@ -131,6 +140,9 @@ public class SubmittedVariantsRestControllerTest extends MongoTestContainerHelpe @MockBean private ContigAliasService contigAliasService; + @Autowired + private DataSource dataSource; + private List> generatedAccessions; private SubmittedVariant variant1; @@ -140,13 +152,21 @@ public class SubmittedVariantsRestControllerTest extends MongoTestContainerHelpe private SubmittedVariant variant3; @BeforeEach - public void setUp() throws AccessionCouldNotBeGeneratedException { + public void setUp() throws AccessionCouldNotBeGeneratedException, SQLException { repository.deleteAll(); mongoTemplate.dropCollection(DbsnpSubmittedVariantEntity.class); mongoTemplate.dropCollection(DbsnpSubmittedVariantOperationEntity.class); mongoTemplate.dropCollection(SubmittedVariantEntity.class); mongoTemplate.dropCollection(SubmittedVariantOperationEntity.class); + try (Connection conn = dataSource.getConnection(); + Statement stmt = conn.createStatement()) { + stmt.execute("DROP TABLE IF EXISTS contiguous_id_blocks"); + } + ResourceDatabasePopulator populator = new ResourceDatabasePopulator(); + populator.addScript(new ClassPathResource("test-data/contiguous_id_blocks_schema.sql")); + populator.execute(dataSource); + Long CLUSTERED_VARIANT = null; variant1 = new SubmittedVariant("ASMACC01", 1101, "PROJACC01", "CHROM1", 1234, "REF", "ALT", CLUSTERED_VARIANT); variant2 = new SubmittedVariant("ASMACC02", 1102, "PROJACC02", "CHROM2", 1234, "REF", "ALT", CLUSTERED_VARIANT); @@ -160,6 +180,8 @@ public void setUp() throws AccessionCouldNotBeGeneratedException { mockController = new SubmittedVariantsRestController(mockService, mockSubmittedVariantsBeaconService); setUpContigAliasMock(); + + } private void setUpContigAliasMock() { diff --git a/eva-accession-ws/src/test/java/uk/ac/ebi/eva/accession/ws/test/MongoTestConfiguration.java b/eva-accession-ws/src/test/java/uk/ac/ebi/eva/accession/ws/test/MongoTestConfiguration.java deleted file mode 100644 index 5a4751e52..000000000 --- a/eva-accession-ws/src/test/java/uk/ac/ebi/eva/accession/ws/test/MongoTestConfiguration.java +++ /dev/null @@ -1,105 +0,0 @@ -/* - * Copyright 2021 EMBL - European Bioinformatics Institute - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package uk.ac.ebi.eva.accession.ws.test; - -import com.mongodb.ReadPreference; -import com.mongodb.WriteConcern; -import com.mongodb.client.MongoClient; -import com.mongodb.client.MongoClients; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.domain.EntityScan; -import org.springframework.boot.test.autoconfigure.data.mongo.AutoConfigureDataMongo; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Primary; -import org.springframework.data.mongodb.MongoDatabaseFactory; -import org.springframework.data.mongodb.config.EnableMongoAuditing; -import org.springframework.data.mongodb.core.MongoTemplate; -import org.springframework.data.mongodb.core.SimpleMongoClientDatabaseFactory; -import org.springframework.data.mongodb.core.WriteResultChecking; -import org.springframework.data.mongodb.core.convert.DefaultDbRefResolver; -import org.springframework.data.mongodb.core.convert.DefaultMongoTypeMapper; -import org.springframework.data.mongodb.core.convert.MappingMongoConverter; -import org.springframework.data.mongodb.core.convert.MongoCustomConversions; -import org.springframework.data.mongodb.core.mapping.MongoMappingContext; -import org.springframework.data.mongodb.repository.config.EnableMongoRepositories; - -import java.util.Collections; - -@Configuration -@EntityScan(basePackages = {"uk.ac.ebi.eva.accession.core.persistence"}) -@EnableMongoRepositories(basePackages = { - "uk.ac.ebi.eva.accession.core.persistence", - "uk.ac.ebi.ampt2d.commons.accession.persistence.mongodb.repository"}) -@EnableMongoAuditing -@AutoConfigureDataMongo -public class MongoTestConfiguration { - - @Value("${spring.data.mongodb.uri}") - private String mongoUri; - - @Value("${spring.data.mongodb.database}") - private String database; - - @Value("${human.mongodb.database}") - private String humanDatabase; - - @Value("${mongodb.read-preference:primary}") - private String readPreference; - - @Bean - @Primary - public MongoClient mongoClient() { - return MongoClients.create(mongoUri); - } - - @Bean("primaryFactory") - @Primary - public MongoDatabaseFactory mongoDbFactory() { - return new SimpleMongoClientDatabaseFactory(mongoClient(), database); - } - - @Bean - @Primary - public MongoCustomConversions mongoCustomConversions() { - return new MongoCustomConversions(Collections.emptyList()); - } - - @Bean - @Primary - public MongoMappingContext mongoMappingContext(MongoCustomConversions conversions) { - MongoMappingContext context = new MongoMappingContext(); - context.setSimpleTypeHolder(conversions.getSimpleTypeHolder()); - context.setAutoIndexCreation(false); - return context; - } - - @Bean - @Primary - public MongoTemplate mongoTemplate(MongoMappingContext mongoMappingContext, - MongoCustomConversions conversions) { - MappingMongoConverter converter = new MappingMongoConverter( - new DefaultDbRefResolver(mongoDbFactory()), mongoMappingContext); - converter.setCustomConversions(conversions); - converter.setTypeMapper(new DefaultMongoTypeMapper(null)); - converter.afterPropertiesSet(); - MongoTemplate mongoTemplate = new MongoTemplate(mongoDbFactory(), converter); - mongoTemplate.setReadPreference(ReadPreference.valueOf(readPreference)); - mongoTemplate.setWriteConcern(WriteConcern.MAJORITY); - mongoTemplate.setWriteResultChecking(WriteResultChecking.EXCEPTION); - return mongoTemplate; - } -} diff --git a/eva-accession-ws/src/test/resources/accession-ws-test.properties b/eva-accession-ws/src/test/resources/accession-ws-test.properties index 82e3e9615..cd3c25b9a 100644 --- a/eva-accession-ws/src/test/resources/accession-ws-test.properties +++ b/eva-accession-ws/src/test/resources/accession-ws-test.properties @@ -10,6 +10,11 @@ accessioning.monotonic.test-rs.blockSize=100000 accessioning.monotonic.test-rs.blockStartValue=3000000000 accessioning.monotonic.test-rs.nextBlockInterval=1000000000 +continuous.id.blocks.datasource.driver-class-name=org.hsqldb.jdbc.JDBCDriver +continuous.id.blocks.datasource.jdbc-url=jdbc:hsqldb:mem:db;sql.syntax_pgs=true;DB_CLOSE_DELAY=-1 +continuous.id.blocks.datasource.username=SA +continuous.id.blocks.datasource.password= + # This symbolic variable is required because # this will be replaced with a default value "localhost" when run locally # (due to the filtering attribute specified in this project's POM) diff --git a/eva-accession-ws/src/test/resources/test-data/contiguous_id_blocks_schema.sql b/eva-accession-ws/src/test/resources/test-data/contiguous_id_blocks_schema.sql new file mode 100644 index 000000000..32dfe369d --- /dev/null +++ b/eva-accession-ws/src/test/resources/test-data/contiguous_id_blocks_schema.sql @@ -0,0 +1,14 @@ +CREATE SEQUENCE IF NOT EXISTS contiguous_id_blocks_seq + START WITH 1 + INCREMENT BY 50; + +CREATE TABLE IF NOT EXISTS contiguous_id_blocks ( + id bigint not NULL, + application_instance_id varchar(255) not NULL, + category_id varchar(255) not NULL, + first_value bigint not NULL, + last_committed bigint not NULL, + last_value bigint not NULL, + reserved boolean NOT NULL, + last_updated_timestamp timestamp NOT NULL +); diff --git a/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/Application.java b/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/Application.java index 41bd1013b..a0bea3cdf 100644 --- a/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/Application.java +++ b/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/Application.java @@ -15,13 +15,25 @@ */ package uk.ac.ebi.eva.remapping.source; -import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.autoconfigure.batch.BatchAutoConfiguration; +import org.springframework.boot.autoconfigure.data.mongo.MongoDataAutoConfiguration; +import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration; +import org.springframework.boot.autoconfigure.jdbc.DataSourceTransactionManagerAutoConfiguration; +import org.springframework.boot.autoconfigure.orm.jpa.HibernateJpaAutoConfiguration; import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.context.annotation.Import; +import uk.ac.ebi.eva.accession.core.configuration.InMemoryBatchConfiguration; -@EnableBatchProcessing -@SpringBootApplication +@SpringBootApplication(exclude = { + MongoDataAutoConfiguration.class, + DataSourceAutoConfiguration.class, + DataSourceTransactionManagerAutoConfiguration.class, + HibernateJpaAutoConfiguration.class, + BatchAutoConfiguration.class +}) +@Import(InMemoryBatchConfiguration.class) public class Application { public static void main(String[] args) { diff --git a/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/configuration/batch/jobs/ExportSubmittedVariantsJobConfiguration.java b/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/configuration/batch/jobs/ExportSubmittedVariantsJobConfiguration.java index c6a982d6a..626d3b2d9 100644 --- a/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/configuration/batch/jobs/ExportSubmittedVariantsJobConfiguration.java +++ b/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/configuration/batch/jobs/ExportSubmittedVariantsJobConfiguration.java @@ -18,31 +18,30 @@ import org.springframework.batch.core.Job; import org.springframework.batch.core.Step; -import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.batch.core.job.builder.JobBuilder; -import org.springframework.batch.core.launch.support.RunIdIncrementer; import org.springframework.batch.core.repository.JobRepository; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Import; -import uk.ac.ebi.eva.remapping.source.configuration.BeanNames; import uk.ac.ebi.eva.remapping.source.configuration.batch.steps.ExportSubmittedVariantsStepConfiguration; +import static uk.ac.ebi.eva.remapping.source.configuration.BeanNames.EXPORT_DBSNP_SUBMITTED_VARIANTS_STEP; +import static uk.ac.ebi.eva.remapping.source.configuration.BeanNames.EXPORT_EVA_SUBMITTED_VARIANTS_STEP; +import static uk.ac.ebi.eva.remapping.source.configuration.BeanNames.EXPORT_SUBMITTED_VARIANTS_JOB; + @Configuration -@EnableBatchProcessing @Import({ExportSubmittedVariantsStepConfiguration.class}) public class ExportSubmittedVariantsJobConfiguration { - @Bean(BeanNames.EXPORT_SUBMITTED_VARIANTS_JOB) + @Bean(EXPORT_SUBMITTED_VARIANTS_JOB) public Job accessionReleaseJob( JobRepository jobRepository, - @Autowired @Qualifier(BeanNames.EXPORT_EVA_SUBMITTED_VARIANTS_STEP) Step exportEvaSubmittedVariantsStep, - @Autowired @Qualifier(BeanNames.EXPORT_DBSNP_SUBMITTED_VARIANTS_STEP) Step exportDbsnpSubmittedVariantsStep + @Autowired @Qualifier(EXPORT_EVA_SUBMITTED_VARIANTS_STEP) Step exportEvaSubmittedVariantsStep, + @Autowired @Qualifier(EXPORT_DBSNP_SUBMITTED_VARIANTS_STEP) Step exportDbsnpSubmittedVariantsStep ) { - return new JobBuilder(BeanNames.EXPORT_SUBMITTED_VARIANTS_JOB, jobRepository) - .incrementer(new RunIdIncrementer()) + return new JobBuilder(EXPORT_SUBMITTED_VARIANTS_JOB, jobRepository) .start(exportEvaSubmittedVariantsStep) .next(exportDbsnpSubmittedVariantsStep) .build(); diff --git a/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/configuration/batch/steps/ExportSubmittedVariantsStepConfiguration.java b/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/configuration/batch/steps/ExportSubmittedVariantsStepConfiguration.java index 14af3acba..f4d956766 100644 --- a/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/configuration/batch/steps/ExportSubmittedVariantsStepConfiguration.java +++ b/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/configuration/batch/steps/ExportSubmittedVariantsStepConfiguration.java @@ -41,6 +41,8 @@ import uk.ac.ebi.eva.remapping.source.configuration.batch.policies.PoliciesConfiguration; import uk.ac.ebi.eva.remapping.source.configuration.batch.processors.SubmittedVariantsProcessorConfiguration; +import static uk.ac.ebi.eva.accession.core.configuration.InMemoryBatchConfiguration.BATCH_TRANSACTION_MANAGER; + @Configuration @Import({SubmittedVariantMongoReaderConfiguration.class, SubmittedVariantsProcessorConfiguration.class, @@ -51,7 +53,8 @@ public class ExportSubmittedVariantsStepConfiguration { @Bean(BeanNames.EXPORT_EVA_SUBMITTED_VARIANTS_STEP) public Step exportEvaSubmittedVariantsStep( - JobRepository jobRepository, PlatformTransactionManager transactionManager, + JobRepository jobRepository, + @Qualifier(BATCH_TRANSACTION_MANAGER) PlatformTransactionManager transactionManager, SimpleCompletionPolicy chunkSizeCompletionPolicy, @Autowired @Qualifier(BeanNames.EVA_SUBMITTED_VARIANT_READER) ItemReader variantReader, @Autowired @Qualifier(BeanNames.SUBMITTED_VARIANT_PROCESSOR) ItemProcessor variantProcessor, @@ -74,7 +77,8 @@ public Step exportEvaSubmittedVariantsStep( @Bean(BeanNames.EXPORT_DBSNP_SUBMITTED_VARIANTS_STEP) public Step exportDbsnpSubmittedVariantsStep( - JobRepository jobRepository, PlatformTransactionManager transactionManager, + JobRepository jobRepository, + @Qualifier(BATCH_TRANSACTION_MANAGER) PlatformTransactionManager transactionManager, SimpleCompletionPolicy chunkSizeCompletionPolicy, @Autowired @Qualifier(BeanNames.DBSNP_SUBMITTED_VARIANT_READER) ItemReader variantReader, @Autowired @Qualifier(BeanNames.SUBMITTED_VARIANT_PROCESSOR) ItemProcessor variantProcessor, diff --git a/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/parameters/InputParameters.java b/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/parameters/InputParameters.java index 4f2d34406..080d01d3b 100644 --- a/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/parameters/InputParameters.java +++ b/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/parameters/InputParameters.java @@ -35,8 +35,6 @@ public class InputParameters { private int taxonomy; - private boolean forceRestart; - private int chunkSize; public JobParameters toJobParameters() { @@ -82,14 +80,6 @@ public void setOutputFolder(String outputFolder) { this.outputFolder = outputFolder; } - public boolean isForceRestart() { - return forceRestart; - } - - public void setForceRestart(boolean forceRestart) { - this.forceRestart = forceRestart; - } - public int getChunkSize() { return chunkSize; } diff --git a/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/runner/AccessionRemappingJobLauncherCommandLineRunner.java b/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/runner/AccessionRemappingJobLauncherCommandLineRunner.java index ee20d70ae..39a7bfbf7 100644 --- a/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/runner/AccessionRemappingJobLauncherCommandLineRunner.java +++ b/eva-remapping-get-source/src/main/java/uk/ac/ebi/eva/remapping/source/runner/AccessionRemappingJobLauncherCommandLineRunner.java @@ -18,7 +18,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.batch.core.Job; -import org.springframework.batch.core.JobExecution; import org.springframework.batch.core.JobExecutionException; import org.springframework.batch.core.JobParameters; import org.springframework.batch.core.JobParametersInvalidException; @@ -35,10 +34,8 @@ import org.springframework.context.ApplicationEventPublisherAware; import org.springframework.stereotype.Component; import org.springframework.util.PatternMatchUtils; -import uk.ac.ebi.eva.accession.core.runner.CommandLineRunnerUtils; import uk.ac.ebi.eva.commons.batch.exception.NoJobToExecuteException; import uk.ac.ebi.eva.commons.batch.exception.NoParametersHaveBeenPassedException; -import uk.ac.ebi.eva.commons.batch.exception.NoPreviousJobExecutionException; import uk.ac.ebi.eva.commons.batch.exception.UnknownJobException; import uk.ac.ebi.eva.commons.batch.job.JobExecutionApplicationListener; import uk.ac.ebi.eva.commons.batch.job.JobStatusManager; @@ -115,18 +112,9 @@ public void run(String... args) throws JobExecutionException { JobStatusManager.checkIfJobNameHasBeenDefined(jobName); JobStatusManager.checkIfPropertiesHaveBeenProvided(jobParameters); - if (inputParameters.isForceRestart()) { - JobExecution previousJobExecution = CommandLineRunnerUtils.getLastJobExecution(jobName, jobExplorer, - jobParameters); - CommandLineRunnerUtils.markPreviousJobAsFailed(jobName, jobRepository, - previousJobExecution != null ? previousJobExecution.getJobParameters() : jobParameters); - } else { - jobParameters = CommandLineRunnerUtils.addRunIDToJobParameters(jobName, jobExplorer, jobParameters); - } - launchJob(jobParameters); - } catch (NoJobToExecuteException | NoParametersHaveBeenPassedException | NoPreviousJobExecutionException - | UnknownJobException | JobParametersInvalidException | JobExecutionAlreadyRunningException e) { + } catch (NoJobToExecuteException | NoParametersHaveBeenPassedException | UnknownJobException + | JobParametersInvalidException | JobExecutionAlreadyRunningException e) { logger.error(e.getMessage()); logger.debug("Error trace", e); abnormalExit = true; diff --git a/eva-remapping-get-source/src/main/resources/application.properties b/eva-remapping-get-source/src/main/resources/application.properties index 5425f1946..8d9ef217b 100644 --- a/eva-remapping-get-source/src/main/resources/application.properties +++ b/eva-remapping-get-source/src/main/resources/application.properties @@ -12,19 +12,10 @@ parameters.taxonomy= parameters.assemblyReportUrl= parameters.outputFolder= -parameters.forceRestart=false # this only affects the frequency of the logs parameters.chunkSize=1000 - -# job repository datasource -spring.datasource.driver-class-name=org.postgresql.Driver -spring.datasource.url= -spring.datasource.username= -spring.datasource.password= -spring.datasource.tomcat.max-active=3 - # Only to set up the database! # spring.jpa.generate-ddl=true diff --git a/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/batch/io/DbsnpSubmittedVariantMongoReaderTest.java b/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/batch/io/DbsnpSubmittedVariantMongoReaderTest.java index c397c49ab..aee09b0e0 100644 --- a/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/batch/io/DbsnpSubmittedVariantMongoReaderTest.java +++ b/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/batch/io/DbsnpSubmittedVariantMongoReaderTest.java @@ -30,9 +30,9 @@ import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.MongoConfiguration; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpSubmittedVariantEntity; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.core.utils.MongoTestDataLoader; -import uk.ac.ebi.eva.remapping.source.test.configuration.MongoTestConfiguration; import java.util.ArrayList; import java.util.Arrays; diff --git a/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/batch/io/EvaSubmittedVariantMongoReaderTest.java b/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/batch/io/EvaSubmittedVariantMongoReaderTest.java index 6925f93e2..18264e977 100644 --- a/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/batch/io/EvaSubmittedVariantMongoReaderTest.java +++ b/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/batch/io/EvaSubmittedVariantMongoReaderTest.java @@ -30,9 +30,9 @@ import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.MongoConfiguration; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.core.utils.MongoTestDataLoader; -import uk.ac.ebi.eva.remapping.source.test.configuration.MongoTestConfiguration; import java.util.ArrayList; import java.util.Collections; diff --git a/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/configuration/batch/jobs/ExportSubmittedVariantsJobConfigurationTest.java b/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/configuration/batch/jobs/ExportSubmittedVariantsJobConfigurationTest.java index 6544bb0c9..6f9212f22 100644 --- a/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/configuration/batch/jobs/ExportSubmittedVariantsJobConfigurationTest.java +++ b/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/configuration/batch/jobs/ExportSubmittedVariantsJobConfigurationTest.java @@ -30,15 +30,14 @@ import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit.jupiter.SpringExtension; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.core.utils.MongoTestDataLoader; import uk.ac.ebi.eva.commons.core.utils.FileUtils; import uk.ac.ebi.eva.remapping.source.configuration.BeanNames; import uk.ac.ebi.eva.remapping.source.parameters.InputParameters; import uk.ac.ebi.eva.remapping.source.parameters.ReportPathResolver; -import uk.ac.ebi.eva.remapping.source.test.configuration.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.remapping.source.test.configuration.BatchTestConfiguration; -import uk.ac.ebi.eva.remapping.source.test.configuration.MongoTestConfiguration; import java.io.FileInputStream; import java.io.FileNotFoundException; @@ -52,8 +51,7 @@ @ExtendWith(SpringExtension.class) @TestPropertySource("classpath:with-projects.properties") -@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class, - BatchJobRepositoryTestConfiguration.class}) +@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class}) public class ExportSubmittedVariantsJobConfigurationTest extends MongoTestContainerHelper { private static final long EXPECTED_LINES_DBSNP = 3; diff --git a/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/runner/AccessionRemappingJobLauncherCommandLineRunnerTest.java b/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/runner/AccessionRemappingJobLauncherCommandLineRunnerTest.java index 358595a01..92856ee8e 100644 --- a/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/runner/AccessionRemappingJobLauncherCommandLineRunnerTest.java +++ b/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/runner/AccessionRemappingJobLauncherCommandLineRunnerTest.java @@ -26,19 +26,17 @@ import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit.jupiter.SpringExtension; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.core.utils.MongoTestDataLoader; import uk.ac.ebi.eva.remapping.source.parameters.InputParameters; import uk.ac.ebi.eva.remapping.source.parameters.ReportPathResolver; -import uk.ac.ebi.eva.remapping.source.test.configuration.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.remapping.source.test.configuration.BatchTestConfiguration; -import uk.ac.ebi.eva.remapping.source.test.configuration.MongoTestConfiguration; import static org.junit.jupiter.api.Assertions.assertEquals; @ExtendWith(SpringExtension.class) -@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class, - BatchJobRepositoryTestConfiguration.class}) +@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class}) @TestPropertySource("classpath:application.properties") public class AccessionRemappingJobLauncherCommandLineRunnerTest extends MongoTestContainerHelper { @Autowired diff --git a/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/runner/AccessionRemappingJobLauncherCommandLineRunnerWithProjectsTest.java b/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/runner/AccessionRemappingJobLauncherCommandLineRunnerWithProjectsTest.java index 37a40d69a..ea64a4fc6 100644 --- a/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/runner/AccessionRemappingJobLauncherCommandLineRunnerWithProjectsTest.java +++ b/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/runner/AccessionRemappingJobLauncherCommandLineRunnerWithProjectsTest.java @@ -26,19 +26,17 @@ import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit.jupiter.SpringExtension; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.core.utils.MongoTestDataLoader; import uk.ac.ebi.eva.remapping.source.parameters.InputParameters; import uk.ac.ebi.eva.remapping.source.parameters.ReportPathResolver; -import uk.ac.ebi.eva.remapping.source.test.configuration.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.remapping.source.test.configuration.BatchTestConfiguration; -import uk.ac.ebi.eva.remapping.source.test.configuration.MongoTestConfiguration; import static org.junit.jupiter.api.Assertions.assertEquals; @ExtendWith(SpringExtension.class) -@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class, - BatchJobRepositoryTestConfiguration.class}) +@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class}) @TestPropertySource("classpath:with-projects.properties") public class AccessionRemappingJobLauncherCommandLineRunnerWithProjectsTest extends MongoTestContainerHelper { diff --git a/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/test/configuration/BatchJobRepositoryTestConfiguration.java b/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/test/configuration/BatchJobRepositoryTestConfiguration.java deleted file mode 100644 index 57c837cc4..000000000 --- a/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/test/configuration/BatchJobRepositoryTestConfiguration.java +++ /dev/null @@ -1,22 +0,0 @@ -package uk.ac.ebi.eva.remapping.source.test.configuration; - -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Primary; -import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseBuilder; -import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseType; - -import javax.sql.DataSource; - -@Configuration -public class BatchJobRepositoryTestConfiguration { - - @Bean - @Primary - public DataSource dataSource() { - return new EmbeddedDatabaseBuilder() - .setType(EmbeddedDatabaseType.HSQL) - .addScript("org/springframework/batch/core/schema-hsqldb.sql") - .build(); - } -} \ No newline at end of file diff --git a/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/test/configuration/BatchTestConfiguration.java b/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/test/configuration/BatchTestConfiguration.java index 9a15f97c3..701d45923 100644 --- a/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/test/configuration/BatchTestConfiguration.java +++ b/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/test/configuration/BatchTestConfiguration.java @@ -21,9 +21,9 @@ import org.springframework.batch.core.repository.JobRepository; import org.springframework.batch.test.JobLauncherTestUtils; import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Import; +import uk.ac.ebi.eva.accession.core.configuration.InMemoryBatchConfiguration; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.MongoConfiguration; import uk.ac.ebi.eva.commons.batch.job.JobExecutionApplicationListener; import uk.ac.ebi.eva.remapping.source.configuration.InputParametersConfiguration; @@ -33,8 +33,8 @@ import static uk.ac.ebi.eva.remapping.source.configuration.BeanNames.EXPORT_SUBMITTED_VARIANTS_JOB; -@EnableAutoConfiguration -@Import({MongoConfiguration.class, +@Import({InMemoryBatchConfiguration.class, + MongoConfiguration.class, InputParametersConfiguration.class, ExportSubmittedVariantsJobConfiguration.class, PoliciesConfiguration.class, diff --git a/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/test/configuration/MongoTestConfiguration.java b/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/test/configuration/MongoTestConfiguration.java deleted file mode 100644 index 1005a6f6a..000000000 --- a/eva-remapping-get-source/src/test/java/uk/ac/ebi/eva/remapping/source/test/configuration/MongoTestConfiguration.java +++ /dev/null @@ -1,105 +0,0 @@ -/* - * - * Copyright 2018 EMBL - European Bioinformatics Institute - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -package uk.ac.ebi.eva.remapping.source.test.configuration; - -import com.mongodb.ReadPreference; -import com.mongodb.WriteConcern; -import com.mongodb.client.MongoClient; -import com.mongodb.client.MongoClients; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.domain.EntityScan; -import org.springframework.boot.test.autoconfigure.data.mongo.AutoConfigureDataMongo; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Primary; -import org.springframework.data.mongodb.MongoDatabaseFactory; -import org.springframework.data.mongodb.config.EnableMongoAuditing; -import org.springframework.data.mongodb.core.MongoTemplate; -import org.springframework.data.mongodb.core.SimpleMongoClientDatabaseFactory; -import org.springframework.data.mongodb.core.WriteResultChecking; -import org.springframework.data.mongodb.core.convert.DefaultDbRefResolver; -import org.springframework.data.mongodb.core.convert.DefaultMongoTypeMapper; -import org.springframework.data.mongodb.core.convert.MappingMongoConverter; -import org.springframework.data.mongodb.core.convert.MongoCustomConversions; -import org.springframework.data.mongodb.core.mapping.MongoMappingContext; -import org.springframework.data.mongodb.repository.config.EnableMongoRepositories; - -import java.util.Collections; - -@Configuration -@EntityScan(basePackages = {"uk.ac.ebi.eva.accession.core.persistence"}) -@EnableMongoRepositories(basePackages = { - "uk.ac.ebi.eva.accession.core.persistence", - "uk.ac.ebi.ampt2d.commons.accession.persistence.mongodb.repository"}) -@EnableMongoAuditing -@AutoConfigureDataMongo -public class MongoTestConfiguration { - - @Value("${spring.data.mongodb.uri}") - private String mongoUri; - - @Value("${spring.data.mongodb.database}") - private String database; - - @Value("${mongodb.read-preference:primary}") - private String readPreference; - - @Bean - @Primary - public MongoClient mongoClient() { - return MongoClients.create(mongoUri); - } - - @Bean("primaryFactory") - @Primary - public MongoDatabaseFactory mongoDbFactory() { - return new SimpleMongoClientDatabaseFactory(mongoClient(), database); - } - - @Bean - @Primary - public MongoCustomConversions mongoCustomConversions() { - return new MongoCustomConversions(Collections.emptyList()); - } - - @Bean - @Primary - public MongoMappingContext mongoMappingContext(MongoCustomConversions conversions) { - MongoMappingContext context = new MongoMappingContext(); - context.setSimpleTypeHolder(conversions.getSimpleTypeHolder()); - context.setAutoIndexCreation(false); - return context; - } - - @Bean - @Primary - public MongoTemplate mongoTemplate(MongoMappingContext mongoMappingContext, - MongoCustomConversions conversions) { - MappingMongoConverter converter = new MappingMongoConverter( - new DefaultDbRefResolver(mongoDbFactory()), mongoMappingContext); - converter.setCustomConversions(conversions); - converter.setTypeMapper(new DefaultMongoTypeMapper(null)); - converter.afterPropertiesSet(); - MongoTemplate mongoTemplate = new MongoTemplate(mongoDbFactory(), converter); - mongoTemplate.setReadPreference(ReadPreference.valueOf(readPreference)); - mongoTemplate.setWriteConcern(WriteConcern.MAJORITY); - mongoTemplate.setWriteResultChecking(WriteResultChecking.EXCEPTION); - return mongoTemplate; - } - -} diff --git a/eva-remapping-get-source/src/test/resources/application.properties b/eva-remapping-get-source/src/test/resources/application.properties index 941cf32a5..cb9e7fe5d 100644 --- a/eva-remapping-get-source/src/test/resources/application.properties +++ b/eva-remapping-get-source/src/test/resources/application.properties @@ -13,7 +13,6 @@ parameters.assemblyAccession=GCA_000409795.2 parameters.fasta=src/test/resources/input-files/fasta/GCF_000409795.2.test.fna parameters.assemblyReportUrl=file:src/test/resources/input-files/assembly-report/GCF_000409795.2_Chlorocebus_sabeus_1.1_assembly_report.txt parameters.outputFolder=/tmp/ -parameters.forceRestart=false parameters.chunkSize=1000 spring.data.mongodb.database=test-db diff --git a/eva-remapping-get-source/src/test/resources/with-projects.properties b/eva-remapping-get-source/src/test/resources/with-projects.properties index d0c6bb68d..9c008b83a 100644 --- a/eva-remapping-get-source/src/test/resources/with-projects.properties +++ b/eva-remapping-get-source/src/test/resources/with-projects.properties @@ -15,7 +15,6 @@ parameters.taxonomy=60711 parameters.fasta=src/test/resources/input-files/fasta/GCF_000409795.2.test.fna parameters.assemblyReportUrl=file:src/test/resources/input-files/assembly-report/GCF_000409795.2_Chlorocebus_sabeus_1.1_assembly_report.txt parameters.outputFolder=/tmp/ -parameters.forceRestart=false parameters.chunkSize=1000 spring.data.mongodb.database=test-db diff --git a/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/Application.java b/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/Application.java index a8e45d5b4..f6c27e636 100644 --- a/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/Application.java +++ b/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/Application.java @@ -15,13 +15,25 @@ */ package uk.ac.ebi.eva.remapping.ingest; -import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.autoconfigure.batch.BatchAutoConfiguration; +import org.springframework.boot.autoconfigure.data.mongo.MongoDataAutoConfiguration; +import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration; +import org.springframework.boot.autoconfigure.jdbc.DataSourceTransactionManagerAutoConfiguration; +import org.springframework.boot.autoconfigure.orm.jpa.HibernateJpaAutoConfiguration; import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.context.annotation.Import; +import uk.ac.ebi.eva.accession.core.configuration.InMemoryBatchConfiguration; -@EnableBatchProcessing -@SpringBootApplication +@SpringBootApplication(exclude = { + MongoDataAutoConfiguration.class, + DataSourceAutoConfiguration.class, + DataSourceTransactionManagerAutoConfiguration.class, + HibernateJpaAutoConfiguration.class, + BatchAutoConfiguration.class +}) +@Import(InMemoryBatchConfiguration.class) public class Application { public static void main(String[] args) { diff --git a/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/jobs/IngestRemappedVariantsFromVcfJobConfiguration.java b/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/jobs/IngestRemappedVariantsFromVcfJobConfiguration.java index d7cee50c2..c0169881f 100644 --- a/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/jobs/IngestRemappedVariantsFromVcfJobConfiguration.java +++ b/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/jobs/IngestRemappedVariantsFromVcfJobConfiguration.java @@ -17,9 +17,7 @@ import org.springframework.batch.core.Job; import org.springframework.batch.core.Step; -import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.batch.core.job.builder.JobBuilder; -import org.springframework.batch.core.launch.support.RunIdIncrementer; import org.springframework.batch.core.repository.JobRepository; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; @@ -30,7 +28,6 @@ import static uk.ac.ebi.eva.remapping.ingest.configuration.BeanNames.STORE_REMAPPING_METADATA_STEP; @Configuration -@EnableBatchProcessing public class IngestRemappedVariantsFromVcfJobConfiguration { @Bean(INGEST_REMAPPED_VARIANTS_FROM_VCF_JOB) @@ -39,7 +36,6 @@ public Job ingestRemappedVariantsFromVcfJob( @Qualifier(STORE_REMAPPING_METADATA_STEP) Step storeRemappingMetadataStep, JobRepository jobRepository) { return new JobBuilder(INGEST_REMAPPED_VARIANTS_FROM_VCF_JOB, jobRepository) - .incrementer(new RunIdIncrementer()) .start(storeRemappingMetadataStep) .next(ingestRemappedVariantsStep) .build(); diff --git a/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/steps/IngestRemappedFromVcfStepConfiguration.java b/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/steps/IngestRemappedFromVcfStepConfiguration.java index e123343ca..98962018d 100644 --- a/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/steps/IngestRemappedFromVcfStepConfiguration.java +++ b/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/steps/IngestRemappedFromVcfStepConfiguration.java @@ -18,7 +18,6 @@ import org.springframework.batch.core.Step; import org.springframework.batch.core.StepExecutionListener; -import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.batch.core.repository.JobRepository; import org.springframework.batch.core.step.builder.StepBuilder; import org.springframework.batch.core.step.tasklet.TaskletStep; @@ -35,13 +34,13 @@ import uk.ac.ebi.eva.commons.core.models.pipeline.Variant; import uk.ac.ebi.eva.remapping.ingest.configuration.BeanNames; +import static uk.ac.ebi.eva.accession.core.configuration.InMemoryBatchConfiguration.BATCH_TRANSACTION_MANAGER; import static uk.ac.ebi.eva.remapping.ingest.configuration.BeanNames.COMPOSITE_VARIANT_PROCESSOR; import static uk.ac.ebi.eva.remapping.ingest.configuration.BeanNames.PROGRESS_LISTENER; import static uk.ac.ebi.eva.remapping.ingest.configuration.BeanNames.REMAPPED_SUBMITTED_VARIANTS_WRITER; import static uk.ac.ebi.eva.remapping.ingest.configuration.BeanNames.VCF_READER; @Configuration -@EnableBatchProcessing public class IngestRemappedFromVcfStepConfiguration { @Bean(BeanNames.INGEST_REMAPPED_VARIANTS_FROM_VCF_STEP) @@ -50,7 +49,8 @@ public Step ingestRemappedFromVcf( @Qualifier(COMPOSITE_VARIANT_PROCESSOR) ItemProcessor processor, @Qualifier(REMAPPED_SUBMITTED_VARIANTS_WRITER) ItemWriter submittedVariantWriter, @Qualifier(PROGRESS_LISTENER) StepExecutionListener progressListener, - JobRepository jobRepository, PlatformTransactionManager transactionManager, + JobRepository jobRepository, + @Qualifier(BATCH_TRANSACTION_MANAGER) PlatformTransactionManager transactionManager, SimpleCompletionPolicy chunkSizeCompletionPolicy) { TaskletStep step = new StepBuilder(BeanNames.INGEST_REMAPPED_VARIANTS_FROM_VCF_STEP, jobRepository) .chunk(chunkSizeCompletionPolicy, transactionManager) diff --git a/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/steps/StoreRemappingMetadataStepConfiguration.java b/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/steps/StoreRemappingMetadataStepConfiguration.java index 6d2c6803b..270b3d254 100644 --- a/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/steps/StoreRemappingMetadataStepConfiguration.java +++ b/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/steps/StoreRemappingMetadataStepConfiguration.java @@ -17,10 +17,10 @@ package uk.ac.ebi.eva.remapping.ingest.configuration.batch.steps; import org.springframework.batch.core.Step; -import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.batch.core.repository.JobRepository; import org.springframework.batch.core.step.builder.StepBuilder; import org.springframework.batch.core.step.tasklet.TaskletStep; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.data.mongodb.core.MongoTemplate; @@ -28,12 +28,10 @@ import uk.ac.ebi.eva.remapping.ingest.batch.tasklets.RemappingMetadata; import uk.ac.ebi.eva.remapping.ingest.batch.tasklets.StoreRemappingMetadataTasklet; -import java.io.IOException; - +import static uk.ac.ebi.eva.accession.core.configuration.InMemoryBatchConfiguration.BATCH_TRANSACTION_MANAGER; import static uk.ac.ebi.eva.remapping.ingest.configuration.BeanNames.STORE_REMAPPING_METADATA_STEP; @Configuration -@EnableBatchProcessing public class StoreRemappingMetadataStepConfiguration { private final MongoTemplate mongoTemplate; @@ -46,7 +44,8 @@ public StoreRemappingMetadataStepConfiguration(MongoTemplate mongoTemplate, Rema } @Bean(STORE_REMAPPING_METADATA_STEP) - public Step buildReportStep(JobRepository jobRepository, PlatformTransactionManager transactionManager) throws IOException { + public Step buildReportStep(JobRepository jobRepository, + @Qualifier(BATCH_TRANSACTION_MANAGER) PlatformTransactionManager transactionManager) { StoreRemappingMetadataTasklet tasklet = new StoreRemappingMetadataTasklet(mongoTemplate, remappingMetadata); TaskletStep step = new StepBuilder(STORE_REMAPPING_METADATA_STEP, jobRepository) .tasklet(tasklet, transactionManager) diff --git a/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/parameters/InputParameters.java b/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/parameters/InputParameters.java index 572b7333b..f0377628e 100644 --- a/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/parameters/InputParameters.java +++ b/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/parameters/InputParameters.java @@ -34,8 +34,6 @@ public class InputParameters { private int chunkSize; - private boolean forceRestart; - public String getVcf() { return vcf; } @@ -92,14 +90,6 @@ public void setChunkSize(int chunkSize) { this.chunkSize = chunkSize; } - public boolean isForceRestart() { - return forceRestart; - } - - public void setForceRestart(boolean forceRestart) { - this.forceRestart = forceRestart; - } - public JobParameters toJobParameters() { return new JobParametersBuilder() .addString("remappedFrom", remappedFrom) diff --git a/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/runner/IngestRemappedCommandLineRunner.java b/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/runner/IngestRemappedCommandLineRunner.java index 021002fd9..e45354396 100644 --- a/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/runner/IngestRemappedCommandLineRunner.java +++ b/eva-remapping-ingest/src/main/java/uk/ac/ebi/eva/remapping/ingest/runner/IngestRemappedCommandLineRunner.java @@ -20,7 +20,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.batch.core.Job; -import org.springframework.batch.core.JobExecution; import org.springframework.batch.core.JobExecutionException; import org.springframework.batch.core.JobParameters; import org.springframework.batch.core.JobParametersInvalidException; @@ -37,10 +36,8 @@ import org.springframework.context.ApplicationEventPublisherAware; import org.springframework.stereotype.Component; import org.springframework.util.PatternMatchUtils; -import uk.ac.ebi.eva.accession.core.runner.CommandLineRunnerUtils; import uk.ac.ebi.eva.commons.batch.exception.NoJobToExecuteException; import uk.ac.ebi.eva.commons.batch.exception.NoParametersHaveBeenPassedException; -import uk.ac.ebi.eva.commons.batch.exception.NoPreviousJobExecutionException; import uk.ac.ebi.eva.commons.batch.exception.UnknownJobException; import uk.ac.ebi.eva.commons.batch.job.JobExecutionApplicationListener; import uk.ac.ebi.eva.commons.batch.job.JobStatusManager; @@ -114,17 +111,10 @@ public void run(String... args) throws JobExecutionException { JobStatusManager.checkIfJobNameHasBeenDefined(jobName); JobStatusManager.checkIfPropertiesHaveBeenProvided(jobParameters); - if (inputParameters.isForceRestart()) { - JobExecution previousJobExecution = CommandLineRunnerUtils.getLastJobExecution(jobName, jobExplorer, - jobParameters); - CommandLineRunnerUtils.markPreviousJobAsFailed(jobName, jobRepository, - previousJobExecution != null ? previousJobExecution.getJobParameters() : jobParameters); - } else { - jobParameters = CommandLineRunnerUtils.addRunIDToJobParameters(jobName, jobExplorer, jobParameters); - } + launchJob(jobParameters); - } catch (NoJobToExecuteException | NoParametersHaveBeenPassedException | NoPreviousJobExecutionException - | UnknownJobException | JobParametersInvalidException | JobExecutionAlreadyRunningException e) { + } catch (NoJobToExecuteException | NoParametersHaveBeenPassedException | UnknownJobException + | JobParametersInvalidException | JobExecutionAlreadyRunningException e) { logger.error(e.getMessage()); logger.debug("Error trace", e); abnormalExit = true; diff --git a/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/batch/io/RemappedSubmittedVariantsWriterTest.java b/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/batch/io/RemappedSubmittedVariantsWriterTest.java index 316e09f93..73a62faf3 100644 --- a/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/batch/io/RemappedSubmittedVariantsWriterTest.java +++ b/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/batch/io/RemappedSubmittedVariantsWriterTest.java @@ -34,10 +34,10 @@ import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantOperationEntity; import uk.ac.ebi.eva.accession.core.summary.SubmittedVariantSummaryFunction; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.remapping.ingest.batch.listeners.RemappingIngestCounts; import uk.ac.ebi.eva.remapping.ingest.test.configuration.BatchTestConfiguration; -import uk.ac.ebi.eva.remapping.ingest.test.configuration.MongoTestConfiguration; import java.time.LocalDateTime; import java.util.Arrays; diff --git a/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/jobs/IngestRemappedVariantsFromVcfJobConfigurationTest.java b/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/jobs/IngestRemappedVariantsFromVcfJobConfigurationTest.java index 232ec7218..656c20c87 100644 --- a/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/jobs/IngestRemappedVariantsFromVcfJobConfigurationTest.java +++ b/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/jobs/IngestRemappedVariantsFromVcfJobConfigurationTest.java @@ -34,12 +34,11 @@ import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.core.utils.MongoTestDataLoader; import uk.ac.ebi.eva.remapping.ingest.batch.tasklets.RemappingMetadata; -import uk.ac.ebi.eva.remapping.ingest.test.configuration.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.remapping.ingest.test.configuration.BatchTestConfiguration; -import uk.ac.ebi.eva.remapping.ingest.test.configuration.MongoTestConfiguration; import java.util.Arrays; import java.util.Collection; @@ -53,8 +52,7 @@ import static uk.ac.ebi.eva.remapping.ingest.test.configuration.BatchTestConfiguration.JOB_INGEST_REMAPPED_VARIANTS_FROM_VCF_JOB; @ExtendWith(SpringExtension.class) -@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class, - BatchJobRepositoryTestConfiguration.class}) +@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class}) @TestPropertySource("classpath:ingest-remapped-variants.properties") public class IngestRemappedVariantsFromVcfJobConfigurationTest extends MongoTestContainerHelper { diff --git a/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/steps/IngestRemappedFromVcfStepConfigurationTest.java b/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/steps/IngestRemappedFromVcfStepConfigurationTest.java index 931ba1175..7fbddaf0c 100644 --- a/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/steps/IngestRemappedFromVcfStepConfigurationTest.java +++ b/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/steps/IngestRemappedFromVcfStepConfigurationTest.java @@ -33,20 +33,18 @@ import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.core.utils.MongoTestDataLoader; import uk.ac.ebi.eva.remapping.ingest.configuration.BeanNames; -import uk.ac.ebi.eva.remapping.ingest.test.configuration.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.remapping.ingest.test.configuration.BatchTestConfiguration; -import uk.ac.ebi.eva.remapping.ingest.test.configuration.MongoTestConfiguration; import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; @ExtendWith(SpringExtension.class) -@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class, - BatchJobRepositoryTestConfiguration.class}) +@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class}) @TestPropertySource("classpath:ingest-remapped-variants.properties") public class IngestRemappedFromVcfStepConfigurationTest extends MongoTestContainerHelper { diff --git a/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/steps/IngestRemappedFromVcfStepDbsnpConfigurationTest.java b/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/steps/IngestRemappedFromVcfStepDbsnpConfigurationTest.java index 377c3ee6d..aa2e218d9 100644 --- a/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/steps/IngestRemappedFromVcfStepDbsnpConfigurationTest.java +++ b/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/configuration/batch/steps/IngestRemappedFromVcfStepDbsnpConfigurationTest.java @@ -33,20 +33,18 @@ import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit.jupiter.SpringExtension; import uk.ac.ebi.eva.accession.core.model.dbsnp.DbsnpSubmittedVariantEntity; +import uk.ac.ebi.eva.accession.core.test.configuration.nonhuman.MongoTestConfiguration; import uk.ac.ebi.eva.accession.core.utils.MongoTestContainerHelper; import uk.ac.ebi.eva.accession.core.utils.MongoTestDataLoader; import uk.ac.ebi.eva.remapping.ingest.configuration.BeanNames; -import uk.ac.ebi.eva.remapping.ingest.test.configuration.BatchJobRepositoryTestConfiguration; import uk.ac.ebi.eva.remapping.ingest.test.configuration.BatchTestConfiguration; -import uk.ac.ebi.eva.remapping.ingest.test.configuration.MongoTestConfiguration; import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; @ExtendWith(SpringExtension.class) -@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class, - BatchJobRepositoryTestConfiguration.class}) +@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class}) @TestPropertySource("classpath:ingest-remapped-variants-dbsnp.properties") public class IngestRemappedFromVcfStepDbsnpConfigurationTest extends MongoTestContainerHelper { diff --git a/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/test/configuration/BatchJobRepositoryTestConfiguration.java b/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/test/configuration/BatchJobRepositoryTestConfiguration.java deleted file mode 100644 index cfa3352cb..000000000 --- a/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/test/configuration/BatchJobRepositoryTestConfiguration.java +++ /dev/null @@ -1,22 +0,0 @@ -package uk.ac.ebi.eva.remapping.ingest.test.configuration; - -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Primary; -import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseBuilder; -import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseType; - -import javax.sql.DataSource; - -@Configuration -public class BatchJobRepositoryTestConfiguration { - - @Bean - @Primary - public DataSource dataSource() { - return new EmbeddedDatabaseBuilder() - .setType(EmbeddedDatabaseType.HSQL) - .addScript("org/springframework/batch/core/schema-hsqldb.sql") - .build(); - } -} \ No newline at end of file diff --git a/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/test/configuration/BatchTestConfiguration.java b/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/test/configuration/BatchTestConfiguration.java index a7065192a..97b49f4d2 100644 --- a/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/test/configuration/BatchTestConfiguration.java +++ b/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/test/configuration/BatchTestConfiguration.java @@ -22,10 +22,10 @@ import org.springframework.batch.test.JobLauncherTestUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Import; import org.springframework.core.io.ResourceLoader; +import uk.ac.ebi.eva.accession.core.configuration.InMemoryBatchConfiguration; import uk.ac.ebi.eva.commons.batch.job.JobExecutionApplicationListener; import uk.ac.ebi.eva.remapping.ingest.configuration.RemappingMetadataConfiguration; import uk.ac.ebi.eva.remapping.ingest.configuration.batch.io.IngestRemappedSubmittedVariantsWriterConfiguration; @@ -39,8 +39,8 @@ import static uk.ac.ebi.eva.remapping.ingest.configuration.BeanNames.INGEST_REMAPPED_VARIANTS_FROM_VCF_JOB; -@EnableAutoConfiguration -@Import({IngestRemappedFromVcfStepConfiguration.class, +@Import({InMemoryBatchConfiguration.class, + IngestRemappedFromVcfStepConfiguration.class, StoreRemappingMetadataStepConfiguration.class, IngestRemappedVariantsFromVcfJobConfiguration.class, VcfReaderConfiguration.class, diff --git a/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/test/configuration/MongoTestConfiguration.java b/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/test/configuration/MongoTestConfiguration.java deleted file mode 100644 index 2cab74199..000000000 --- a/eva-remapping-ingest/src/test/java/uk/ac/ebi/eva/remapping/ingest/test/configuration/MongoTestConfiguration.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Copyright 2021 EMBL - European Bioinformatics Institute - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package uk.ac.ebi.eva.remapping.ingest.test.configuration; - -import com.mongodb.ReadPreference; -import com.mongodb.WriteConcern; -import com.mongodb.client.MongoClient; -import com.mongodb.client.MongoClients; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.domain.EntityScan; -import org.springframework.boot.test.autoconfigure.data.mongo.AutoConfigureDataMongo; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Primary; -import org.springframework.data.mongodb.MongoDatabaseFactory; -import org.springframework.data.mongodb.config.EnableMongoAuditing; -import org.springframework.data.mongodb.core.MongoTemplate; -import org.springframework.data.mongodb.core.SimpleMongoClientDatabaseFactory; -import org.springframework.data.mongodb.core.WriteResultChecking; -import org.springframework.data.mongodb.core.convert.DefaultDbRefResolver; -import org.springframework.data.mongodb.core.convert.DefaultMongoTypeMapper; -import org.springframework.data.mongodb.core.convert.MappingMongoConverter; -import org.springframework.data.mongodb.core.convert.MongoCustomConversions; -import org.springframework.data.mongodb.core.mapping.MongoMappingContext; -import org.springframework.data.mongodb.repository.config.EnableMongoRepositories; - -import java.util.Collections; - -@Configuration -@EntityScan(basePackages = {"uk.ac.ebi.eva.accession.core.persistence"}) -@EnableMongoRepositories(basePackages = { - "uk.ac.ebi.eva.accession.core.persistence", - "uk.ac.ebi.ampt2d.commons.accession.persistence.mongodb.repository"}) -@EnableMongoAuditing -@AutoConfigureDataMongo -public class MongoTestConfiguration { - - @Value("${spring.data.mongodb.uri}") - private String mongoUri; - - @Value("${spring.data.mongodb.database}") - private String database; - - @Value("${mongodb.read-preference:primary}") - private String readPreference; - - @Bean - @Primary - public MongoClient mongoClient() { - return MongoClients.create(mongoUri); - } - - @Bean("primaryFactory") - @Primary - public MongoDatabaseFactory mongoDbFactory() { - return new SimpleMongoClientDatabaseFactory(mongoClient(), database); - } - - @Bean - @Primary - public MongoCustomConversions mongoCustomConversions() { - return new MongoCustomConversions(Collections.emptyList()); - } - - @Bean - @Primary - public MongoMappingContext mongoMappingContext(MongoCustomConversions conversions) { - MongoMappingContext context = new MongoMappingContext(); - context.setSimpleTypeHolder(conversions.getSimpleTypeHolder()); - context.setAutoIndexCreation(false); - return context; - } - - @Bean - @Primary - public MongoTemplate mongoTemplate(MongoMappingContext mongoMappingContext, - MongoCustomConversions conversions) { - MappingMongoConverter converter = new MappingMongoConverter( - new DefaultDbRefResolver(mongoDbFactory()), mongoMappingContext); - converter.setCustomConversions(conversions); - converter.setTypeMapper(new DefaultMongoTypeMapper(null)); - converter.afterPropertiesSet(); - MongoTemplate mongoTemplate = new MongoTemplate(mongoDbFactory(), converter); - mongoTemplate.setReadPreference(ReadPreference.valueOf(readPreference)); - mongoTemplate.setWriteConcern(WriteConcern.MAJORITY); - mongoTemplate.setWriteResultChecking(WriteResultChecking.EXCEPTION); - return mongoTemplate; - } -} diff --git a/pom.xml b/pom.xml index 9843ab4d1..58224787b 100644 --- a/pom.xml +++ b/pom.xml @@ -153,17 +153,6 @@ test - - org.postgresql - postgresql - - - - org.hsqldb - hsqldb - test - - org.springframework.boot spring-boot-testcontainers From 0579f0b05abc22586dbf8735349a3a1dfb0685d5 Mon Sep 17 00:00:00 2001 From: nitin-ebi <79518737+nitin-ebi@users.noreply.github.com> Date: Mon, 8 Jun 2026 09:43:23 +0100 Subject: [PATCH 09/10] EVA-4113 Integration test updates (#503) --- .../configuration/RunnerConfiguration.java | 52 +++++++++++++++++++ .../pipeline/parameters/InputParameters.java | 12 +++-- 2 files changed, 60 insertions(+), 4 deletions(-) create mode 100644 eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/RunnerConfiguration.java diff --git a/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/RunnerConfiguration.java b/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/RunnerConfiguration.java new file mode 100644 index 000000000..321e73f15 --- /dev/null +++ b/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/RunnerConfiguration.java @@ -0,0 +1,52 @@ +/* + * Copyright 2018 EMBL - European Bioinformatics Institute + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package uk.ac.ebi.eva.accession.deprecate.configuration; + +import org.springframework.batch.core.ExitStatus; +import org.springframework.batch.core.Job; +import org.springframework.batch.core.JobExecution; +import org.springframework.batch.core.JobParametersBuilder; +import org.springframework.batch.core.launch.JobLauncher; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.CommandLineRunner; +import org.springframework.context.ApplicationContext; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import uk.ac.ebi.eva.commons.batch.job.JobExecutionApplicationListener; + +@Configuration +public class RunnerConfiguration { + + @Bean + public JobExecutionApplicationListener jobExecutionApplicationListener() { + return new JobExecutionApplicationListener(); + } + + @Bean + public CommandLineRunner runJob(JobLauncher jobLauncher, ApplicationContext context, + @Value("${spring.batch.job.names}") String jobName) { + return args -> { + Job job = context.getBean(jobName, Job.class); + JobExecution execution = jobLauncher.run(job, new JobParametersBuilder() + .toJobParameters()); + + if (!execution.getExitStatus().equals(ExitStatus.COMPLETED)) { + throw new RuntimeException("Job failed with status: " + execution.getExitStatus()); + } + }; + } +} \ No newline at end of file diff --git a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/parameters/InputParameters.java b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/parameters/InputParameters.java index 1f6e017e0..beca7cb77 100644 --- a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/parameters/InputParameters.java +++ b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/parameters/InputParameters.java @@ -47,18 +47,22 @@ public class InputParameters { private String duplicateSSAccFile; public JobParameters toJobParameters() { - return new JobParametersBuilder() + JobParametersBuilder builder = new JobParametersBuilder() .addString("vcf", vcf) .addString("vcfAggregation", vcfAggregation.toString()) - .addString("aggregatedMappingFile", aggregatedMappingFile) .addString("fasta", fasta) .addString("outputVcf", outputVcf) .addLong("taxonomyAccession", (long) taxonomyAccession) .addString("assemblyAccession", assemblyAccession) .addString("projectAccession", projectAccession) .addLong("chunkSize", (long) chunkSize) - .addString("contigNaming", contigNaming.toString()) - .toJobParameters(); + .addString("contigNaming", contigNaming.toString()); + + if (aggregatedMappingFile != null) { + builder.addString("aggregatedMappingFile", aggregatedMappingFile); + } + + return builder.toJobParameters(); } public String getVcf() { From cddeed28c9b3b3953572fc6959a52cd171130c43 Mon Sep 17 00:00:00 2001 From: nkumar2 Date: Tue, 9 Jun 2026 00:42:43 +0100 Subject: [PATCH 10/10] Bumping version to 1.0.0-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 58224787b..c7898199b 100644 --- a/pom.xml +++ b/pom.xml @@ -22,7 +22,7 @@ - 0.6.58-SNAPSHOT + 1.0.0-SNAPSHOT 21 UTF-8 1.0.0-SNAPSHOT