Drop stale funders when acknowledgement overlap cleanup removes their annotation

grobid-core/src/main/java/org/grobid/core/data/AnnotatedXMLElement.java

@@ -28,11 +28,24 @@ public class AnnotatedXMLElement {
     private OffsetPosition offsetPosition;
     private Element annotationNode;
 
+    /**
+     * The extracted entity this annotation was derived from (a Funding, Person or Affiliation).
+     * Kept so that, when an annotation cannot be injected back into the XML, the corresponding
+     * entity can be identified and dropped by reference instead of by fragile text matching.
+     */
+    private Object entity;
+
     public AnnotatedXMLElement(Element annotationNode, OffsetPosition offsetPosition) {
         this.annotationNode = annotationNode;
         this.offsetPosition = offsetPosition;
     }
 
+    public AnnotatedXMLElement(Element annotationNode, OffsetPosition offsetPosition, Object entity) {
+        this.annotationNode = annotationNode;
+        this.offsetPosition = offsetPosition;
+        this.entity = entity;
+    }
+
     public OffsetPosition getOffsetPosition() {
         return offsetPosition;
     }
@@ -48,4 +61,12 @@ public Element getAnnotationNode() {
     public void setAnnotationNode(Element annotationNode) {
         this.annotationNode = annotationNode;
     }
+
+    public Object getEntity() {
+        return entity;
+    }
+
+    public void setEntity(Object entity) {
+        this.entity = entity;
+    }
 }

grobid-core/src/main/java/org/grobid/core/engines/FundingAcknowledgementParser.java

@@ -224,26 +224,33 @@ public MutablePair<Element, MutableTriple<List<Funding>, List<Person>, List<Affi
                 for (int i = 0; i < annotationsPositionText.size(); i++) {
                     annotationsWithPosRefToText.add(
                             new AnnotatedXMLElement(annotations.get(i).getAnnotationNode(),
-                                    annotationsPositionText.get(i)));
+                                    annotationsPositionText.get(i),
+                                    annotations.get(i).getEntity()));
                 }
 
                 annotations = annotationsWithPosRefToText;
 
+                Set<AnnotatedXMLElement> injectedAnnotations = Collections.newSetFromMap(new IdentityHashMap<>());
                 if (sentenceSegmentation) {
                     Nodes sentences = paragraph.query(".//s");
 
                     if (sentences.size() == 0) {
                         // Overly careful - we should never end up here.
                         LOGGER.warn(
                                 "While the configuration claim that paragraphs must be segmented, we did not find any sentence. ");
-                        updateParagraphNodeWithAnnotations(paragraph, annotations);
+                        injectedAnnotations.addAll(updateParagraphNodeWithAnnotations(paragraph, annotations));
                     }
                     mergeSentencesFallingOnAnnotations(sentences, annotations, config);
-                    updateSentencesNodesWithAnnotations(sentences, annotations);
+                    injectedAnnotations.addAll(updateSentencesNodesWithAnnotations(sentences, annotations));
                 } else {
-                    updateParagraphNodeWithAnnotations(paragraph, annotations);
+                    injectedAnnotations.addAll(updateParagraphNodeWithAnnotations(paragraph, annotations));
                 }
 
+                // Drop the extracted entities whose annotations could not be injected back into the
+                // XML (e.g. because they overlap a pre-existing inline element): keeping them would
+                // yield entities with no corresponding inline annotation in the output.
+                pruneEntitiesWithoutInjectedAnnotation(localEntities, annotations, injectedAnnotations);
+
                 // update extracted entities
                 if (globalResult == null) {
                     globalResult = MutablePair.of(
@@ -409,7 +416,10 @@ private static List<OffsetPosition> getOffsetPositionsFromNodes(Nodes sentences)
         return sentencePositions;
     }
 
-    private static void updateParagraphNodeWithAnnotations(Node paragraph, List<AnnotatedXMLElement> annotations) {
+    private static Set<AnnotatedXMLElement> updateParagraphNodeWithAnnotations(
+            Node paragraph,
+            List<AnnotatedXMLElement> annotations) {
+        Set<AnnotatedXMLElement> injectedAnnotations = Collections.newSetFromMap(new IdentityHashMap<>());
         int pos = 0;
         List<Node> newChildren = new ArrayList<>();
         for (int i = 0; i < paragraph.getChildCount(); i++) {
@@ -427,6 +437,7 @@ private static void updateParagraphNodeWithAnnotations(Node paragraph, List<Anno
                 if (CollectionUtils.isNotEmpty(annotationsInThisChunk)) {
                     List<Node> nodes = getNodesAnnotationsInTextNode(currentNode, annotationsInThisChunk, pos);
                     newChildren.addAll(nodes);
+                    injectedAnnotations.addAll(annotationsInThisChunk);
                 } else {
                     newChildren.add(currentNode);
                 }
@@ -444,9 +455,14 @@ private static void updateParagraphNodeWithAnnotations(Node paragraph, List<Anno
             node.detach();
             ((Element) paragraph).appendChild(node);
         }
+
+        return injectedAnnotations;
     }
 
-    private static void updateSentencesNodesWithAnnotations(Nodes sentences, List<AnnotatedXMLElement> annotations) {
+    private static Set<AnnotatedXMLElement> updateSentencesNodesWithAnnotations(
+            Nodes sentences,
+            List<AnnotatedXMLElement> annotations) {
+        Set<AnnotatedXMLElement> injectedAnnotations = Collections.newSetFromMap(new IdentityHashMap<>());
         int pos = 0;
         int sentenceStartOffset = 0;
         for (Node sentence : sentences) {
@@ -467,6 +483,7 @@ private static void updateSentencesNodesWithAnnotations(Nodes sentences, List<An
                     if (CollectionUtils.isNotEmpty(annotationsInThisChunk)) {
                         List<Node> nodes = getNodesAnnotationsInTextNode(currentNode, annotationsInThisChunk, pos);
                         newChildren.addAll(nodes);
+                        injectedAnnotations.addAll(annotationsInThisChunk);
                     } else {
                         newChildren.add(currentNode);
                     }
@@ -489,6 +506,73 @@ private static void updateSentencesNodesWithAnnotations(Nodes sentences, List<An
 
             sentenceStartOffset += sentenceText.length();
         }
+
+        return injectedAnnotations;
+    }
+
+    /**
+     * Removes the extracted entities (fundings, persons, affiliations) whose annotations were all
+     * dropped during injection into the XML. An entity is kept when it has no recorded annotation
+     * at all, or when at least one of its annotations was successfully injected; it is dropped only
+     * when it had annotations and none of them survived. The correspondence is resolved by object
+     * identity through {@link AnnotatedXMLElement#getEntity()}, so entities sharing the same textual
+     * value (e.g. two fundings with the same funder name) remain distinct.
+     */
+    private static void pruneEntitiesWithoutInjectedAnnotation(
+            FundingAcknowledgmentParse localEntities,
+            List<AnnotatedXMLElement> allAnnotations,
+            Set<AnnotatedXMLElement> injectedAnnotations) {
+        if (localEntities == null) {
+            return;
+        }
+
+        Set<Object> entitiesWithAnyAnnotation = Collections.newSetFromMap(new IdentityHashMap<>());
+        for (AnnotatedXMLElement annotation : allAnnotations) {
+            if (annotation.getEntity() != null) {
+                entitiesWithAnyAnnotation.add(annotation.getEntity());
+            }
+        }
+
+        Set<Object> entitiesWithInjectedAnnotation = Collections.newSetFromMap(new IdentityHashMap<>());
+        for (AnnotatedXMLElement annotation : injectedAnnotations) {
+            if (annotation.getEntity() != null) {
+                entitiesWithInjectedAnnotation.add(annotation.getEntity());
+            }
+        }
+
+        localEntities.setFundings(
+                retainEntitiesWithInjectedAnnotation(
+                        localEntities.getFundings(),
+                        entitiesWithAnyAnnotation,
+                        entitiesWithInjectedAnnotation));
+        localEntities.setPersons(
+                retainEntitiesWithInjectedAnnotation(
+                        localEntities.getPersons(),
+                        entitiesWithAnyAnnotation,
+                        entitiesWithInjectedAnnotation));
+        localEntities.setAffiliations(
+                retainEntitiesWithInjectedAnnotation(
+                        localEntities.getAffiliations(),
+                        entitiesWithAnyAnnotation,
+                        entitiesWithInjectedAnnotation));
+    }
+
+    private static <T> List<T> retainEntitiesWithInjectedAnnotation(
+            List<T> entities,
+            Set<Object> entitiesWithAnyAnnotation,
+            Set<Object> entitiesWithInjectedAnnotation) {
+        if (CollectionUtils.isEmpty(entities)) {
+            return entities;
+        }
+
+        List<T> retained = new ArrayList<>();
+        for (T entity : entities) {
+            if (!entitiesWithAnyAnnotation.contains(entity)
+                    || entitiesWithInjectedAnnotation.contains(entity)) {
+                retained.add(entity);
+            }
+        }
+        return retained;
     }
 
     /**
@@ -628,6 +712,8 @@ protected MutablePair<List<AnnotatedXMLElement>, FundingAcknowledgmentParse> get
 
         List<Element> elements = new ArrayList<>();
         List<OffsetPosition> positions = new ArrayList<>();
+        // the extracted entity each annotation belongs to, aligned with elements/positions
+        List<Object> owners = new ArrayList<>();
 
         int posTokenization = 0;
         int posCharacters = 0;
@@ -693,6 +779,7 @@ protected MutablePair<List<AnnotatedXMLElement>, FundingAcknowledgmentParse> get
                 entity.addAttribute(new Attribute("type", "funder"));
                 entity.appendChild(clusterContent);
                 elements.add(entity);
+                owners.add(funding);
 
                 positions.add(new OffsetPosition(posTokenization, endPosTokenization));
             } else if (clusterLabel.equals(FUNDING_GRANT_NAME)) {
@@ -712,6 +799,7 @@ protected MutablePair<List<AnnotatedXMLElement>, FundingAcknowledgmentParse> get
                 entity.addAttribute(new Attribute("type", "grantName"));
                 entity.appendChild(clusterContent);
                 elements.add(entity);
+                owners.add(funding);
 
                 positions.add(new OffsetPosition(posTokenization, endPosTokenization));
 
@@ -731,6 +819,7 @@ protected MutablePair<List<AnnotatedXMLElement>, FundingAcknowledgmentParse> get
                 entity.addAttribute(new Attribute("type", "person"));
                 entity.appendChild(clusterContent);
                 elements.add(entity);
+                owners.add(person);
 
                 positions.add(new OffsetPosition(posTokenization, endPosTokenization));
 
@@ -750,6 +839,7 @@ protected MutablePair<List<AnnotatedXMLElement>, FundingAcknowledgmentParse> get
                 entity.addAttribute(new Attribute("type", "affiliation"));
                 entity.appendChild(clusterContent);
                 elements.add(entity);
+                owners.add(affiliation);
 
                 positions.add(new OffsetPosition(posTokenization, endPosTokenization));
 
@@ -769,6 +859,7 @@ protected MutablePair<List<AnnotatedXMLElement>, FundingAcknowledgmentParse> get
                 entity.addAttribute(new Attribute("type", "institution"));
                 entity.appendChild(clusterContent);
                 elements.add(entity);
+                owners.add(institution);
 
                 positions.add(new OffsetPosition(posTokenization, endPosTokenization));
 
@@ -789,6 +880,7 @@ protected MutablePair<List<AnnotatedXMLElement>, FundingAcknowledgmentParse> get
                 entity.addAttribute(new Attribute("subtype", "infrastructure"));
                 entity.appendChild(clusterContent);
                 elements.add(entity);
+                owners.add(institution);
 
                 positions.add(new OffsetPosition(posTokenization, endPosTokenization));
 
@@ -818,6 +910,7 @@ protected MutablePair<List<AnnotatedXMLElement>, FundingAcknowledgmentParse> get
                 entity.addAttribute(new Attribute("type", "grantNumber"));
                 entity.appendChild(clusterContent);
                 elements.add(entity);
+                owners.add(funding);
 
                 positions.add(new OffsetPosition(posTokenization, endPosTokenization));
 
@@ -838,6 +931,7 @@ protected MutablePair<List<AnnotatedXMLElement>, FundingAcknowledgmentParse> get
                 entity.addAttribute(new Attribute("type", "programName"));
                 entity.appendChild(clusterContent);
                 elements.add(entity);
+                owners.add(funding);
 
                 positions.add(new OffsetPosition(posTokenization, endPosTokenization));
 
@@ -858,6 +952,7 @@ protected MutablePair<List<AnnotatedXMLElement>, FundingAcknowledgmentParse> get
                 entity.addAttribute(new Attribute("type", "projectName"));
                 entity.appendChild(clusterContent);
                 elements.add(entity);
+                owners.add(funding);
 
                 positions.add(new OffsetPosition(posTokenization, endPosTokenization));
 
@@ -896,7 +991,7 @@ protected MutablePair<List<AnnotatedXMLElement>, FundingAcknowledgmentParse> get
         List<AnnotatedXMLElement> annotations = new ArrayList<>();
 
         for (int i = 0; i < elements.size(); i++) {
-            annotations.add(new AnnotatedXMLElement(elements.get(i), positions.get(i)));
+            annotations.add(new AnnotatedXMLElement(elements.get(i), positions.get(i), owners.get(i)));
         }
 
         return MutablePair.of(annotations, parsedStatement);

grobid-core/src/test/kotlin/org/grobid/core/engines/FundingAcknowledgementParserIntegrationTest.kt

@@ -15,10 +15,13 @@
  */
 package org.grobid.core.engines
 
+import org.grobid.core.GrobidModels
 import org.grobid.core.engines.config.GrobidAnalysisConfig
 import org.grobid.core.factory.AbstractEngineFactory
 import org.grobid.core.utilities.GrobidConfig
 import org.grobid.core.utilities.GrobidProperties
+import org.hamcrest.CoreMatchers.containsString
+import org.hamcrest.CoreMatchers.not
 import org.hamcrest.MatcherAssert.assertThat
 import org.hamcrest.Matchers.hasSize
 import org.junit.Before
@@ -275,6 +278,73 @@ class FundingAcknowledgementParserIntegrationTest {
         assertThat(element.toXML(), CompareMatcher.isIdenticalTo(output))
     }
 
+    @Test
+    fun testXmlFragmentProcessing_shouldDropFundingWithoutInjectableAnnotation() {
+        val input = """
+            <div type="acknowledgement">
+            <div xmlns="http://www.tei-c.org/ns/1.0"><head>Acknowledgements</head><p><s>provide public access to these results of federally sponsored research in accordance with the DOE Public Access Plan <ref type="url" target="https://example.org/public-access">http</ref></s></p></div>
+            </div>
+        """.trimIndent()
+
+        val parser = StubFundingAcknowledgementParser(
+            listOf("DOE", "Public", "Access", "Plan", "http"),
+        )
+
+        val config = GrobidAnalysisConfig.GrobidAnalysisConfigBuilder()
+            .withSentenceSegmentation(true)
+            .build()
+
+        val (element, mutableTriple) = parser.processingXmlFragment(input, config)
+
+        assertThat(mutableTriple.left, hasSize(0))
+        assertThat(element.toXML(), not(containsString("type=\"funder\"")))
+    }
+
+    @Test
+    fun testXmlFragmentProcessing_shouldKeepFundingWithInjectableAnnotation() {
+        // The funder span falls entirely within plain text (no overlapping inline element),
+        // so its annotation can be injected and the funding must be retained.
+        val input = """
+            <div type="acknowledgement">
+            <div xmlns="http://www.tei-c.org/ns/1.0"><head>Acknowledgements</head><p><s>This work was supported by the National Science Foundation</s></p></div>
+            </div>
+        """.trimIndent()
+
+        val parser = StubFundingAcknowledgementParser(
+            listOf("National", "Science", "Foundation"),
+        )
+
+        val config = GrobidAnalysisConfig.GrobidAnalysisConfigBuilder()
+            .withSentenceSegmentation(true)
+            .build()
+
+        val (element, mutableTriple) = parser.processingXmlFragment(input, config)
+
+        assertThat(mutableTriple.left, hasSize(1))
+        assertThat(element.toXML(), containsString("type=\"funder\""))
+    }
+
+    private class StubFundingAcknowledgementParser(
+        private val tokensToLabelAsFunder: List<String>,
+    ) : FundingAcknowledgementParser(GrobidModels.DUMMY) {
+        override fun label(data: String): String {
+            val lines = data.lineSequence()
+                .filter { it.isNotBlank() }
+                .toList()
+            val start = (lines.size - tokensToLabelAsFunder.size).coerceAtLeast(0)
+
+            return lines.mapIndexed { index, line ->
+                val label = when {
+                    index == start -> "I-<funderName>"
+                    index in (start + 1) until (start + tokensToLabelAsFunder.size) -> "<funderName>"
+                    index == 0 -> "I-<other>"
+                    else -> "<other>"
+                }
+                "$line\t$label"
+            }.joinToString("\n")
+        }
+    }
+
     companion object {
         @JvmStatic
         @BeforeClass