diff --git a/grobid-core/src/main/java/org/grobid/core/data/AnnotatedXMLElement.java b/grobid-core/src/main/java/org/grobid/core/data/AnnotatedXMLElement.java index 8de9ffbc26..8288dac27b 100644 --- a/grobid-core/src/main/java/org/grobid/core/data/AnnotatedXMLElement.java +++ b/grobid-core/src/main/java/org/grobid/core/data/AnnotatedXMLElement.java @@ -28,11 +28,24 @@ public class AnnotatedXMLElement { private OffsetPosition offsetPosition; private Element annotationNode; + /** + * The extracted entity this annotation was derived from (a Funding, Person or Affiliation). + * Kept so that, when an annotation cannot be injected back into the XML, the corresponding + * entity can be identified and dropped by reference instead of by fragile text matching. + */ + private Object entity; + public AnnotatedXMLElement(Element annotationNode, OffsetPosition offsetPosition) { this.annotationNode = annotationNode; this.offsetPosition = offsetPosition; } + public AnnotatedXMLElement(Element annotationNode, OffsetPosition offsetPosition, Object entity) { + this.annotationNode = annotationNode; + this.offsetPosition = offsetPosition; + this.entity = entity; + } + public OffsetPosition getOffsetPosition() { return offsetPosition; } @@ -48,4 +61,12 @@ public Element getAnnotationNode() { public void setAnnotationNode(Element annotationNode) { this.annotationNode = annotationNode; } + + public Object getEntity() { + return entity; + } + + public void setEntity(Object entity) { + this.entity = entity; + } } diff --git a/grobid-core/src/main/java/org/grobid/core/engines/FundingAcknowledgementParser.java b/grobid-core/src/main/java/org/grobid/core/engines/FundingAcknowledgementParser.java index 6909b3b955..26a7dca6e5 100644 --- a/grobid-core/src/main/java/org/grobid/core/engines/FundingAcknowledgementParser.java +++ b/grobid-core/src/main/java/org/grobid/core/engines/FundingAcknowledgementParser.java @@ -224,11 +224,13 @@ public MutablePair, List, List injectedAnnotations = Collections.newSetFromMap(new IdentityHashMap<>()); if (sentenceSegmentation) { Nodes sentences = paragraph.query(".//s"); @@ -236,14 +238,19 @@ public MutablePair, List, List getOffsetPositionsFromNodes(Nodes sentences) return sentencePositions; } - private static void updateParagraphNodeWithAnnotations(Node paragraph, List annotations) { + private static Set updateParagraphNodeWithAnnotations( + Node paragraph, + List annotations) { + Set injectedAnnotations = Collections.newSetFromMap(new IdentityHashMap<>()); int pos = 0; List newChildren = new ArrayList<>(); for (int i = 0; i < paragraph.getChildCount(); i++) { @@ -427,6 +437,7 @@ private static void updateParagraphNodeWithAnnotations(Node paragraph, List nodes = getNodesAnnotationsInTextNode(currentNode, annotationsInThisChunk, pos); newChildren.addAll(nodes); + injectedAnnotations.addAll(annotationsInThisChunk); } else { newChildren.add(currentNode); } @@ -444,9 +455,14 @@ private static void updateParagraphNodeWithAnnotations(Node paragraph, List annotations) { + private static Set updateSentencesNodesWithAnnotations( + Nodes sentences, + List annotations) { + Set injectedAnnotations = Collections.newSetFromMap(new IdentityHashMap<>()); int pos = 0; int sentenceStartOffset = 0; for (Node sentence : sentences) { @@ -467,6 +483,7 @@ private static void updateSentencesNodesWithAnnotations(Nodes sentences, List nodes = getNodesAnnotationsInTextNode(currentNode, annotationsInThisChunk, pos); newChildren.addAll(nodes); + injectedAnnotations.addAll(annotationsInThisChunk); } else { newChildren.add(currentNode); } @@ -489,6 +506,73 @@ private static void updateSentencesNodesWithAnnotations(Nodes sentences, List allAnnotations, + Set injectedAnnotations) { + if (localEntities == null) { + return; + } + + Set entitiesWithAnyAnnotation = Collections.newSetFromMap(new IdentityHashMap<>()); + for (AnnotatedXMLElement annotation : allAnnotations) { + if (annotation.getEntity() != null) { + entitiesWithAnyAnnotation.add(annotation.getEntity()); + } + } + + Set entitiesWithInjectedAnnotation = Collections.newSetFromMap(new IdentityHashMap<>()); + for (AnnotatedXMLElement annotation : injectedAnnotations) { + if (annotation.getEntity() != null) { + entitiesWithInjectedAnnotation.add(annotation.getEntity()); + } + } + + localEntities.setFundings( + retainEntitiesWithInjectedAnnotation( + localEntities.getFundings(), + entitiesWithAnyAnnotation, + entitiesWithInjectedAnnotation)); + localEntities.setPersons( + retainEntitiesWithInjectedAnnotation( + localEntities.getPersons(), + entitiesWithAnyAnnotation, + entitiesWithInjectedAnnotation)); + localEntities.setAffiliations( + retainEntitiesWithInjectedAnnotation( + localEntities.getAffiliations(), + entitiesWithAnyAnnotation, + entitiesWithInjectedAnnotation)); + } + + private static List retainEntitiesWithInjectedAnnotation( + List entities, + Set entitiesWithAnyAnnotation, + Set entitiesWithInjectedAnnotation) { + if (CollectionUtils.isEmpty(entities)) { + return entities; + } + + List retained = new ArrayList<>(); + for (T entity : entities) { + if (!entitiesWithAnyAnnotation.contains(entity) + || entitiesWithInjectedAnnotation.contains(entity)) { + retained.add(entity); + } + } + return retained; } /** @@ -628,6 +712,8 @@ protected MutablePair, FundingAcknowledgmentParse> get List elements = new ArrayList<>(); List positions = new ArrayList<>(); + // the extracted entity each annotation belongs to, aligned with elements/positions + List owners = new ArrayList<>(); int posTokenization = 0; int posCharacters = 0; @@ -693,6 +779,7 @@ protected MutablePair, FundingAcknowledgmentParse> get entity.addAttribute(new Attribute("type", "funder")); entity.appendChild(clusterContent); elements.add(entity); + owners.add(funding); positions.add(new OffsetPosition(posTokenization, endPosTokenization)); } else if (clusterLabel.equals(FUNDING_GRANT_NAME)) { @@ -712,6 +799,7 @@ protected MutablePair, FundingAcknowledgmentParse> get entity.addAttribute(new Attribute("type", "grantName")); entity.appendChild(clusterContent); elements.add(entity); + owners.add(funding); positions.add(new OffsetPosition(posTokenization, endPosTokenization)); @@ -731,6 +819,7 @@ protected MutablePair, FundingAcknowledgmentParse> get entity.addAttribute(new Attribute("type", "person")); entity.appendChild(clusterContent); elements.add(entity); + owners.add(person); positions.add(new OffsetPosition(posTokenization, endPosTokenization)); @@ -750,6 +839,7 @@ protected MutablePair, FundingAcknowledgmentParse> get entity.addAttribute(new Attribute("type", "affiliation")); entity.appendChild(clusterContent); elements.add(entity); + owners.add(affiliation); positions.add(new OffsetPosition(posTokenization, endPosTokenization)); @@ -769,6 +859,7 @@ protected MutablePair, FundingAcknowledgmentParse> get entity.addAttribute(new Attribute("type", "institution")); entity.appendChild(clusterContent); elements.add(entity); + owners.add(institution); positions.add(new OffsetPosition(posTokenization, endPosTokenization)); @@ -789,6 +880,7 @@ protected MutablePair, FundingAcknowledgmentParse> get entity.addAttribute(new Attribute("subtype", "infrastructure")); entity.appendChild(clusterContent); elements.add(entity); + owners.add(institution); positions.add(new OffsetPosition(posTokenization, endPosTokenization)); @@ -818,6 +910,7 @@ protected MutablePair, FundingAcknowledgmentParse> get entity.addAttribute(new Attribute("type", "grantNumber")); entity.appendChild(clusterContent); elements.add(entity); + owners.add(funding); positions.add(new OffsetPosition(posTokenization, endPosTokenization)); @@ -838,6 +931,7 @@ protected MutablePair, FundingAcknowledgmentParse> get entity.addAttribute(new Attribute("type", "programName")); entity.appendChild(clusterContent); elements.add(entity); + owners.add(funding); positions.add(new OffsetPosition(posTokenization, endPosTokenization)); @@ -858,6 +952,7 @@ protected MutablePair, FundingAcknowledgmentParse> get entity.addAttribute(new Attribute("type", "projectName")); entity.appendChild(clusterContent); elements.add(entity); + owners.add(funding); positions.add(new OffsetPosition(posTokenization, endPosTokenization)); @@ -896,7 +991,7 @@ protected MutablePair, FundingAcknowledgmentParse> get List annotations = new ArrayList<>(); for (int i = 0; i < elements.size(); i++) { - annotations.add(new AnnotatedXMLElement(elements.get(i), positions.get(i))); + annotations.add(new AnnotatedXMLElement(elements.get(i), positions.get(i), owners.get(i))); } return MutablePair.of(annotations, parsedStatement); diff --git a/grobid-core/src/test/kotlin/org/grobid/core/engines/FundingAcknowledgementParserIntegrationTest.kt b/grobid-core/src/test/kotlin/org/grobid/core/engines/FundingAcknowledgementParserIntegrationTest.kt index c292bf9c57..8bd12e8807 100644 --- a/grobid-core/src/test/kotlin/org/grobid/core/engines/FundingAcknowledgementParserIntegrationTest.kt +++ b/grobid-core/src/test/kotlin/org/grobid/core/engines/FundingAcknowledgementParserIntegrationTest.kt @@ -15,10 +15,13 @@ */ package org.grobid.core.engines +import org.grobid.core.GrobidModels import org.grobid.core.engines.config.GrobidAnalysisConfig import org.grobid.core.factory.AbstractEngineFactory import org.grobid.core.utilities.GrobidConfig import org.grobid.core.utilities.GrobidProperties +import org.hamcrest.CoreMatchers.containsString +import org.hamcrest.CoreMatchers.not import org.hamcrest.MatcherAssert.assertThat import org.hamcrest.Matchers.hasSize import org.junit.Before @@ -275,6 +278,73 @@ class FundingAcknowledgementParserIntegrationTest { assertThat(element.toXML(), CompareMatcher.isIdenticalTo(output)) } + @Test + fun testXmlFragmentProcessing_shouldDropFundingWithoutInjectableAnnotation() { + val input = """ +
+
Acknowledgements

provide public access to these results of federally sponsored research in accordance with the DOE Public Access Plan http

+
+ """.trimIndent() + + val parser = StubFundingAcknowledgementParser( + listOf("DOE", "Public", "Access", "Plan", "http"), + ) + + val config = GrobidAnalysisConfig.GrobidAnalysisConfigBuilder() + .withSentenceSegmentation(true) + .build() + + val (element, mutableTriple) = parser.processingXmlFragment(input, config) + + assertThat(mutableTriple.left, hasSize(0)) + assertThat(element.toXML(), not(containsString("type=\"funder\""))) + } + + @Test + fun testXmlFragmentProcessing_shouldKeepFundingWithInjectableAnnotation() { + // The funder span falls entirely within plain text (no overlapping inline element), + // so its annotation can be injected and the funding must be retained. + val input = """ +
+
Acknowledgements

This work was supported by the National Science Foundation

+
+ """.trimIndent() + + val parser = StubFundingAcknowledgementParser( + listOf("National", "Science", "Foundation"), + ) + + val config = GrobidAnalysisConfig.GrobidAnalysisConfigBuilder() + .withSentenceSegmentation(true) + .build() + + val (element, mutableTriple) = parser.processingXmlFragment(input, config) + + assertThat(mutableTriple.left, hasSize(1)) + assertThat(element.toXML(), containsString("type=\"funder\"")) + } + + private class StubFundingAcknowledgementParser( + private val tokensToLabelAsFunder: List, + ) : FundingAcknowledgementParser(GrobidModels.DUMMY) { + override fun label(data: String): String { + val lines = data.lineSequence() + .filter { it.isNotBlank() } + .toList() + val start = (lines.size - tokensToLabelAsFunder.size).coerceAtLeast(0) + + return lines.mapIndexed { index, line -> + val label = when { + index == start -> "I-" + index in (start + 1) until (start + tokensToLabelAsFunder.size) -> "" + index == 0 -> "I-" + else -> "" + } + "$line\t$label" + }.joinToString("\n") + } + } + companion object { @JvmStatic @BeforeClass