EVA-4113 Update pipeline to Java21

.github/workflows/run-tests.yml

@@ -2,44 +2,28 @@ name: Run tests
 
 on:
   push:
-    branches: [ master ]
+    branches: [ master, java21 ]
   pull_request:
-    # Temporarily run tests on the feature branch until merged to master
-    branches: [ master, feature/ws-contig-alias-integration ]
+    branches: [ master, java21 ]
 
 jobs:
   build:
     runs-on: ubuntu-latest
+
     strategy:
       matrix:
-        # Including "ubuntu2204" in the version is a temporary fix to the direct download URL while we transition
-        # between 4 and 6
-        mongodb-version: [4.0.18, ubuntu2204-6.0.6]
-        java-version: [1.8]
-        python-version: [3.8]
+        java-version: [21]
 
     steps:
-      - uses: actions/checkout@v2
-      - name: Setup Java JDK
-        uses: actions/setup-java@v1.4.3
-        # The Java version to make available on the path. Takes a whole or semver Java version, or 1.x syntax (e.g. 1.8 => Java 8.x). Early access versions can be specified in the form of e.g. 14-ea, 14.0.0-ea, or 14.0.0-ea.28
-        with:
-          java-version: ${{ matrix.java-version }}      
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Java 21
+        uses: actions/setup-java@v4
         with:
-          python-version: ${{ matrix.python-version }}
-      - name: Install Python dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install numpy pandas psycopg2-binary pgpasslib
-      - name: Install and Start MongoDB
-        run: |
-          wget https://fastdl.mongodb.org/linux/mongodb-linux-x86_64-${{ matrix.mongodb-version }}.tgz
-          tar xfz mongodb-linux-x86_64-${{ matrix.mongodb-version }}.tgz
-          export PATH=`pwd`/mongodb-linux-x86_64-${{ matrix.mongodb-version }}/bin:$PATH
-          mkdir -p data/db
-          mongod --dbpath=data/db &
-          mongod --version
+          distribution: temurin
+          java-version: ${{ matrix.java-version }}
+          cache: maven
+
       - name: Run Maven tests
-        run: mvn test
+        run: mvn -B clean test

.gitlab-ci.yml

@@ -28,7 +28,7 @@ test:
         - mongo:4.0.18
     script:
         # Gitlab exposes services under their own hostnames. So test host should be "mongo" instead of "localhost".
-        - mvn clean test --projects 'eva-accession-core,eva-accession-deprecate,eva-accession-ws,eva-remapping-get-source,eva-accession-clustering,eva-accession-pipeline,eva-accession-release,eva-remapping-ingest' -Deva.mongo.host.test=mongo
+        - mvn clean test --projects 'eva-accession-core,eva-accession-deprecate,eva-accession-ws,eva-remapping-get-source,eva-accession-clustering,eva-accession-pipeline,eva-accession-release,eva-remapping-ingest'
     environment:
         name: test-env
     only:

eva-accession-clustering/pom.xml

@@ -2,12 +2,13 @@
 <project xmlns="http://maven.apache.org/POM/4.0.0"
          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
     <parent>
         <artifactId>eva-accession</artifactId>
         <groupId>uk.ac.ebi.eva</groupId>
-        <version>0.6.58-SNAPSHOT</version>
+        <version>${revision}</version>
     </parent>
-    <modelVersion>4.0.0</modelVersion>
 
     <artifactId>eva-accession-clustering</artifactId>
     <packaging>jar</packaging>
@@ -31,25 +32,59 @@
         </dependency>
 
         <dependency>
-            <groupId>com.google.guava</groupId>
-            <artifactId>guava</artifactId>
-            <version>33.4.0-jre</version>
+            <groupId>com.google.code.gson</groupId>
+            <artifactId>gson</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>org.postgresql</groupId>
+            <artifactId>postgresql</artifactId>
         </dependency>
 
         <dependency>
             <groupId>com.thoughtworks.xstream</groupId>
             <artifactId>xstream</artifactId>
         </dependency>
 
+        <dependency>
+            <groupId>uk.ac.ebi.eva</groupId>
+            <artifactId>eva-accession-core</artifactId>
+            <version>${revision}</version>
+            <type>test-jar</type>
+            <scope>test</scope>
+        </dependency>
+
         <dependency>
             <groupId>org.springframework.batch</groupId>
             <artifactId>spring-batch-test</artifactId>
             <scope>test</scope>
         </dependency>
 
         <dependency>
-            <groupId>com.lordofthejars</groupId>
-            <artifactId>nosqlunit-mongodb</artifactId>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-test</artifactId>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.hsqldb</groupId>
+            <artifactId>hsqldb</artifactId>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-testcontainers</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.testcontainers</groupId>
+            <artifactId>mongodb</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.testcontainers</groupId>
+            <artifactId>junit-jupiter</artifactId>
             <scope>test</scope>
         </dependency>
     </dependencies>
@@ -76,9 +111,19 @@
 
         <testResources>
             <testResource>
-                <directory>src/test/resources/</directory>
+                <directory>src/test/resources</directory>
+                <excludes>
+                    <exclude>**/*.gz</exclude>
+                </excludes>
                 <filtering>true</filtering>
             </testResource>
+            <testResource>
+                <directory>src/test/resources</directory>
+                <includes>
+                    <include>**/*.gz</include>
+                </includes>
+                <filtering>false</filtering>
+            </testResource>
         </testResources>
 
         <resources>
@@ -90,8 +135,4 @@
 
     </build>
 
-    <properties>
-        <eva.mongo.host.test>localhost</eva.mongo.host.test>
-    </properties>
-
 </project>

eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/Application.java

@@ -15,13 +15,26 @@
  */
 package uk.ac.ebi.eva.accession.clustering;
 
-import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing;
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.boot.autoconfigure.batch.BatchAutoConfiguration;
+import org.springframework.boot.autoconfigure.data.mongo.MongoDataAutoConfiguration;
+import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration;
+import org.springframework.boot.autoconfigure.jdbc.DataSourceTransactionManagerAutoConfiguration;
+import org.springframework.boot.autoconfigure.orm.jpa.HibernateJpaAutoConfiguration;
 import org.springframework.context.ConfigurableApplicationContext;
+import org.springframework.context.annotation.Import;
+import uk.ac.ebi.eva.accession.core.configuration.ContiguousIdBlocksDataSourceConfiguration;
+import uk.ac.ebi.eva.accession.core.configuration.InMemoryBatchConfiguration;
 
-@EnableBatchProcessing
-@SpringBootApplication
+@SpringBootApplication(exclude = {
+        MongoDataAutoConfiguration.class,
+        DataSourceAutoConfiguration.class,
+        DataSourceTransactionManagerAutoConfiguration.class,
+        HibernateJpaAutoConfiguration.class,
+        BatchAutoConfiguration.class
+})
+@Import({InMemoryBatchConfiguration.class, ContiguousIdBlocksDataSourceConfiguration.class})
 public class Application {
 
     public static void main(String[] args) {

eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/io/BackPropagatedRSWriter.java

@@ -16,13 +16,13 @@
 package uk.ac.ebi.eva.accession.clustering.batch.io;
 
 import com.mongodb.MongoBulkWriteException;
-
+import jakarta.annotation.Nonnull;
+import org.springframework.batch.item.Chunk;
 import org.springframework.batch.item.ItemWriter;
 import org.springframework.data.mongodb.core.BulkOperations;
 import org.springframework.data.mongodb.core.MongoTemplate;
 import org.springframework.data.mongodb.core.query.Query;
 import org.springframework.data.mongodb.core.query.Update;
-
 import uk.ac.ebi.ampt2d.commons.accession.core.exceptions.AccessionCouldNotBeGeneratedException;
 import uk.ac.ebi.ampt2d.commons.accession.core.models.EventType;
 import uk.ac.ebi.ampt2d.commons.accession.persistence.mongodb.document.AccessionedDocument;
@@ -36,7 +36,6 @@
 import uk.ac.ebi.eva.accession.core.service.nonhuman.SubmittedVariantAccessioningService;
 import uk.ac.ebi.eva.metrics.metric.MetricCompute;
 
-import javax.annotation.Nonnull;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.List;
@@ -74,11 +73,10 @@ public BackPropagatedRSWriter(String remappedAssembly, ClusteringWriter clusteri
     }
 
     @Override
-    public void write(
-            @Nonnull List<? extends SubmittedVariantEntity> submittedVariantEntitiesInOriginalAssembly)
+    public void write(@Nonnull Chunk<? extends SubmittedVariantEntity> submittedVariantEntitiesInOriginalAssembly)
             throws MongoBulkWriteException, AccessionCouldNotBeGeneratedException {
         List<SubmittedVariantEntity> ssToLookupInRemappedAssembly =
-                submittedVariantEntitiesInOriginalAssembly
+                submittedVariantEntitiesInOriginalAssembly.getItems()
                         .stream()
                         // Some dbSNP imported variants might have been explicitly de-clustered
                         // because the REF/ALT allele data provided by dbSNP was internally inconsistent
@@ -89,15 +87,15 @@ public void write(
                         .collect(Collectors.toList());
         List<Long> ssIDsToLookupInRemappedAssembly =
                 ssToLookupInRemappedAssembly.stream().map(AccessionedDocument::getAccession)
-                                            .collect(Collectors.toList());
+                        .collect(Collectors.toList());
 
         Map<Long, List<SubmittedVariantEntity>> ssInRemappedAssemblyGroupedByID =
                 submittedVariantAccessioningService
                         .getAllActiveByAssemblyAndAccessionIn(remappedAssembly, ssIDsToLookupInRemappedAssembly)
                         .stream()
                         .map(entity -> new SubmittedVariantEntity(entity.getAccession(), entity.getHash(),
-                                                                  entity.getData(),
-                                                                  entity.getVersion()))
+                                entity.getData(),
+                                entity.getVersion()))
                         .filter(entity -> Objects.nonNull(entity.getClusteredVariantAccession()))
                         .collect(Collectors.groupingBy(SubmittedVariantEntity::getAccession));
         backpropagateRSToSS(ssToLookupInRemappedAssembly, ssInRemappedAssemblyGroupedByID);
@@ -109,21 +107,21 @@ private void backpropagateRSToSS(
 
         // Updates to Submitted Variant Entity (SVE) collection with the RS created above
         BulkOperations evaSVEUpdates = mongoTemplate.bulkOps(BulkOperations.BulkMode.UNORDERED,
-                                                             SubmittedVariantEntity.class);
+                SubmittedVariantEntity.class);
         BulkOperations dbsnpSVEUpdates = mongoTemplate.bulkOps(BulkOperations.BulkMode.UNORDERED,
-                                                               DbsnpSubmittedVariantEntity.class);
+                DbsnpSubmittedVariantEntity.class);
 
         // Inserts to Submitted Variant Operation Entity (SVOE) collection recording the RS updates made above
         BulkOperations evaSVOEInserts = mongoTemplate.bulkOps(BulkOperations.BulkMode.UNORDERED,
-                                                              SubmittedVariantOperationEntity.class);
+                SubmittedVariantOperationEntity.class);
         BulkOperations dbsnpSVOEInserts = mongoTemplate.bulkOps(BulkOperations.BulkMode.UNORDERED,
-                                                                DbsnpSubmittedVariantOperationEntity.class);
+                DbsnpSubmittedVariantOperationEntity.class);
 
 
         int numEvaRSAssignments = 0;
         int numDbsnpRSAssignments = 0;
 
-        for (SubmittedVariantEntity submittedVariantEntity: submittedVariantEntitiesInOriginalAssemblyWithNoRS) {
+        for (SubmittedVariantEntity submittedVariantEntity : submittedVariantEntitiesInOriginalAssemblyWithNoRS) {
             Long ssIDToBeClustered = submittedVariantEntity.getAccession();
             Long rsInOriginalAssembly = submittedVariantEntity.getClusteredVariantAccession();
             Long rsToBackPropagate = null;