diff --git a/grobid-core/src/main/java/org/grobid/core/document/DocumentNode.java b/grobid-core/src/main/java/org/grobid/core/document/DocumentNode.java index 9a3845b6c2..864437a3ed 100755 --- a/grobid-core/src/main/java/org/grobid/core/document/DocumentNode.java +++ b/grobid-core/src/main/java/org/grobid/core/document/DocumentNode.java @@ -18,10 +18,12 @@ import java.util.ArrayList; import java.util.List; +import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.builder.ToStringBuilder; import org.apache.commons.lang3.builder.ToStringStyle; import org.grobid.core.layout.BoundingBox; +import org.grobid.core.utilities.TextUtilities; /** * Class corresponding to a node of the structure of a hierarchically organized document (i.e. for a table @@ -222,4 +224,73 @@ public Integer getId() { public void setId(Integer id) { this.id = id; } + + // similarity threshold above which an outline label is considered to match a section head + private static final double NODE_MATCH_THRESHOLD = 0.90; + + /** + * Normalize an outline label or a section head for soft comparison. Outline labels extracted + * from the PDF bookmarks carry non-breaking spaces, soft hyphens, '|' number separators and + * multi-line indentation runs, none of which are meaningful for matching. + */ + private static String normalizeForMatching(String value) { + if (value == null) { + return null; + } + String normalized = value + .replace('\u00A0', ' ') // non-breaking space + .replace("\u00AD", "") // soft hyphen + .replace('|', ' '); + return StringUtils.normalizeSpace(normalized); + } + + /** + * Given the label of a potential node (typically a section head), find it in the hierarchy + * rooted at {@code rootNode} using soft string matching and return its depth. + * + * @param rootNode the node to search from + * @param label the section head text to look up + * @param depth the depth of {@code rootNode} in the hierarchy (0 for the outline root) + * @return the depth of the first matching node, or -1 if no node matches + */ + public static int findNodeDepth(DocumentNode rootNode, String label, int depth) { + DocumentNode node = findNode(rootNode, label); + if (node == null) { + return -1; + } + int d = depth; + DocumentNode cursor = node; + while (cursor != rootNode && cursor.getFather() != null) { + d++; + cursor = cursor.getFather(); + } + return d; + } + + /** + * Find the first node in the hierarchy rooted at {@code rootNode} whose label softly matches + * {@code label} (depth-first). Returns the matching node itself, or null if none matches. + */ + public static DocumentNode findNode(DocumentNode rootNode, String label) { + if (rootNode == null || label == null) { + return null; + } + String normalizedLabel = normalizeForMatching(label); + String nodeLabel = normalizeForMatching(rootNode.getLabel()); + if (StringUtils.isNotBlank(nodeLabel)) { + double score = TextUtilities.getRatcliffObershelpSimilarity(nodeLabel, normalizedLabel, false); + if (score >= NODE_MATCH_THRESHOLD) { + return rootNode; + } + } + if (rootNode.getChildren() != null) { + for (DocumentNode child : rootNode.getChildren()) { + DocumentNode found = findNode(child, label); + if (found != null) { + return found; + } + } + } + return null; + } } diff --git a/grobid-core/src/main/java/org/grobid/core/document/TEIFormatter.java b/grobid-core/src/main/java/org/grobid/core/document/TEIFormatter.java index c493a1bd12..2783e2892f 100755 --- a/grobid-core/src/main/java/org/grobid/core/document/TEIFormatter.java +++ b/grobid-core/src/main/java/org/grobid/core/document/TEIFormatter.java @@ -1609,6 +1609,84 @@ public StringBuilder toTEIAnnex( return buffer; } + /** + * Per-head hierarchy level derived from the PDF outline: 1 for a main section, 2 for a + * sub-section, and so on. Keyed by cluster identity (not head text, which can repeat across a + * document). {@code active} is true when the outline gives enough signal to annotate this piece. + */ + static class SectionLevelInfo { + final Map levels; + final boolean active; + + SectionLevelInfo(Map levels, boolean active) { + this.levels = levels; + this.active = active; + } + } + + /** + * Compute, for each SECTION head that can be located in the document outline, its hierarchy level: + * 1 for the shallowest matched section, +1 for each outline level below it. The outline is the + * only signal that reveals when a section returns to a higher level (the "ascent"), which cannot + * be inferred from the text alone. Activated only when the outline exists, the piece has at least + * one pair of adjacent section heads (evidence of hierarchy), and at least one head matched. + */ + static SectionLevelInfo computeSectionLevels(List clusters, DocumentNode outlineRoot) { + Map depths = new IdentityHashMap<>(); + int minDepth = Integer.MAX_VALUE; + + boolean hasAdjacentSectionHeads = false; + TaggingLabel previousLabel = null; + for (TaggingTokenCluster cluster : clusters) { + if (cluster == null) { + continue; + } + TaggingLabel clusterLabel = cluster.getTaggingLabel(); + if (TaggingLabels.SECTION.equals(clusterLabel) && TaggingLabels.SECTION.equals(previousLabel)) { + hasAdjacentSectionHeads = true; + } + previousLabel = clusterLabel; + } + + if (hasAdjacentSectionHeads && outlineRoot != null) { + for (TaggingTokenCluster cluster : clusters) { + if (cluster == null || !TaggingLabels.SECTION.equals(cluster.getTaggingLabel())) { + continue; + } + String clusterContent = LayoutTokensUtil.normalizeDehyphenizeText(cluster.concatTokens()); + int depth = findOutlineDepthForHead(outlineRoot, clusterContent); + if (depth > 0) { + depths.put(cluster, depth); + minDepth = Math.min(minDepth, depth); + } + } + } + + Map levels = new IdentityHashMap<>(); + for (Map.Entry entry : depths.entrySet()) { + levels.put(entry.getKey(), entry.getValue() - minDepth + 1); + } + boolean active = hasAdjacentSectionHeads && !levels.isEmpty(); + return new SectionLevelInfo(levels, active); + } + + /** + * Depth of the outline node matching a section head, or -1 if none. Tries the head text as + * labelled first (keeping the section number to disambiguate same-titled sections), then retries + * with the number stripped, since many outlines store the bare title ("Results") while the head + * carries a number ("2. Results") that pushes a short title below the similarity threshold. + */ + static int findOutlineDepthForHead(DocumentNode outlineRoot, String headText) { + int depth = DocumentNode.findNodeDepth(outlineRoot, headText, 0); + if (depth < 0) { + org.grobid.core.utilities.Pair numb = getSectionNumber(headText); + if (numb != null && StringUtils.isNotBlank(numb.a)) { + depth = DocumentNode.findNodeDepth(outlineRoot, numb.a, 0); + } + } + return depth; + } + public StringBuilder toTEITextPiece( StringBuilder buffer, String result, @@ -1638,6 +1716,11 @@ public StringBuilder toTEITextPiece( List divResults = new ArrayList<>(); + // When the PDF provides an outline (table of content), annotate each section head with its + // hierarchy level (1 = main section, 2 = sub-section, ...). The div structure is left flat and + // identical to the outline-less output; only a @level attribute is added on matched heads. + SectionLevelInfo sectionLevelInfo = computeSectionLevels(clusters, doc.getOutlineRoot()); + Element curDiv = teiElement("div"); if (config.isGenerateTeiIds()) { String divID = KeyGen.getKey().substring(0, 7); @@ -1668,6 +1751,12 @@ public StringBuilder toTEITextPiece( head.appendChild(clusterContent); } + // hierarchy level from the PDF outline, when this head could be located in it + Integer level = sectionLevelInfo.levels.get(cluster); + if (level != null) { + head.addAttribute(new Attribute("level", String.valueOf(level))); + } + if (config.isGenerateTeiIds()) { String divID = KeyGen.getKey().substring(0, 7); addXmlId(head, "_" + divID); @@ -2381,7 +2470,7 @@ private List getGraphicObject(List graphicObjects, return result; } - private org.grobid.core.utilities.Pair getSectionNumber(String text) { + private static org.grobid.core.utilities.Pair getSectionNumber(String text) { Matcher m1 = BasicStructureBuilder.headerNumbering1.matcher(text); Matcher m2 = BasicStructureBuilder.headerNumbering2.matcher(text); Matcher m3 = BasicStructureBuilder.headerNumbering3.matcher(text); diff --git a/grobid-core/src/main/java/org/grobid/core/engines/FullTextParser.java b/grobid-core/src/main/java/org/grobid/core/engines/FullTextParser.java index 9e84d8df67..85b9c47ea3 100755 --- a/grobid-core/src/main/java/org/grobid/core/engines/FullTextParser.java +++ b/grobid-core/src/main/java/org/grobid/core/engines/FullTextParser.java @@ -99,7 +99,7 @@ public Document processing(File inputPdf, GrobidAnalysisConfig config) throws Exception { DocumentSource documentSource = DocumentSource.fromPdf(inputPdf, config.getStartPage(), config.getEndPage(), - config.getPdfAssetPath() != null, true, false); + config.getPdfAssetPath() != null, true, true); documentSource.setMD5(md5Str); return processing(documentSource, flavor, config); } diff --git a/grobid-core/src/main/java/org/grobid/core/sax/PDFALTOOutlineSaxHandler.java b/grobid-core/src/main/java/org/grobid/core/sax/PDFALTOOutlineSaxHandler.java index 459000e39d..f437b79786 100644 --- a/grobid-core/src/main/java/org/grobid/core/sax/PDFALTOOutlineSaxHandler.java +++ b/grobid-core/src/main/java/org/grobid/core/sax/PDFALTOOutlineSaxHandler.java @@ -44,7 +44,9 @@ public class PDFALTOOutlineSaxHandler extends DefaultHandler { private int currentLevel = -1; private int currentId = -1; - private int currentParentId = -1; + // parent id of the TOCITEMLIST currently open, one entry per nesting level; + // -1 when the list has no idItemParent (root list) + private Deque currentParentIdStack = new ArrayDeque<>(); private Map nodes = null; @@ -80,7 +82,12 @@ public void endElement( box = null; label = null; } else if (qName.equals("TOCITEMLIST")) { - currentParentId = -1; + if (!currentParentIdStack.isEmpty()) { + currentParentIdStack.pop(); + } else { + LOGGER.warn( + "TOCITEMLIST end encountered with empty parent stack. Possible malformed outline structure."); + } } else if (qName.equals("LINK")) { // in case of nested item, we need to assign the box right away or we will lose it. if (box != null) { @@ -123,7 +130,8 @@ public void startElement( } currentNode.setId(currentId); nodes.put(currentId, currentNode); - if (currentParentId != -1) { + Integer currentParentId = currentParentIdStack.peek(); + if (currentParentId != null && currentParentId != -1) { DocumentNode father = nodes.get(currentParentId); if (father == null) LOGGER.warn("Father not yet encountered! id is " + currentParentId); @@ -141,6 +149,9 @@ public void startElement( // we only consider annotation with attribute @subtype of value "Link" int length = atts.getLength(); + // the root TOCITEMLIST has no idItemParent attribute + int parentId = -1; + // Process attributes for (int i = 0; i < length; i++) { // Get names and values for each attribute @@ -157,14 +168,16 @@ public void startElement( } } else if (name.equals("idItemParent")) { try { - currentParentId = Integer.parseInt(value); + parentId = Integer.parseInt(value); } catch (Exception e) { LOGGER.warn("Invalid parent id string (should be an integer): " + value); - currentParentId = -1; + parentId = -1; } } } } + // always push exactly one entry per TOCITEMLIST so that start/end events stay balanced + currentParentIdStack.push(parentId); } else if (qName.equals("LINK")) { int length = atts.getLength(); diff --git a/grobid-core/src/test/java/org/grobid/core/document/TEIFormatterTest.java b/grobid-core/src/test/java/org/grobid/core/document/TEIFormatterTest.java index c3b4ec4cc4..a43816abea 100644 --- a/grobid-core/src/test/java/org/grobid/core/document/TEIFormatterTest.java +++ b/grobid-core/src/test/java/org/grobid/core/document/TEIFormatterTest.java @@ -32,7 +32,11 @@ import org.grobid.core.data.Figure; import org.grobid.core.data.Note; import org.grobid.core.data.Table; +import org.grobid.core.document.TEIFormatter.SectionLevelInfo; +import org.grobid.core.engines.label.TaggingLabels; import org.grobid.core.layout.LayoutToken; +import org.grobid.core.tokenization.LabeledTokensContainer; +import org.grobid.core.tokenization.TaggingTokenCluster; import org.grobid.core.utilities.GrobidProperties; import org.grobid.core.utilities.LayoutTokensUtil; @@ -423,4 +427,129 @@ public void testMarkReferencesTableTEI_truncatedRef2_referenceAtBeginning() thro assertThat(nodes.get(5).toXML(), is(" ")); } + private static TaggingTokenCluster sectionCluster(String text) { + List tokens = GrobidAnalyzer.getInstance().tokenizeWithLayoutToken(text); + TaggingTokenCluster cluster = new TaggingTokenCluster(TaggingLabels.SECTION); + cluster.addLabeledTokensContainer( + new LabeledTokensContainer(tokens, text, TaggingLabels.SECTION, true)); + return cluster; + } + + private static TaggingTokenCluster paragraphCluster(String text) { + List tokens = GrobidAnalyzer.getInstance().tokenizeWithLayoutToken(text); + TaggingTokenCluster cluster = new TaggingTokenCluster(TaggingLabels.PARAGRAPH); + cluster.addLabeledTokensContainer( + new LabeledTokensContainer(tokens, text, TaggingLabels.PARAGRAPH, true)); + return cluster; + } + + // outline: root -> title -> { "2 Methods" -> "2.1 Data", "3 Results" -> "3.1 Findings" } + private static DocumentNode buildOutline() { + DocumentNode root = new DocumentNode(); + DocumentNode title = new DocumentNode("Some Article Title", null); + DocumentNode methods = new DocumentNode("2 Methods", null); + DocumentNode data = new DocumentNode("2.1 Data", null); + DocumentNode results = new DocumentNode("3 Results", null); + DocumentNode findings = new DocumentNode("3.1 Findings", null); + root.addChild(title); + title.addChild(methods); + methods.addChild(data); + title.addChild(results); + results.addChild(findings); + return root; + } + + @Test + public void testComputeSectionLevels_inactiveWithoutAdjacentHeads() { + TaggingTokenCluster methods = sectionCluster("2 Methods"); + TaggingTokenCluster para = paragraphCluster("Some body text."); + TaggingTokenCluster data = sectionCluster("2.1 Data"); + + SectionLevelInfo info = TEIFormatter.computeSectionLevels( + List.of(methods, para, data), + buildOutline()); + + // no two section heads are adjacent, so the mechanism stays off + assertThat(info.active, is(false)); + } + + @Test + public void testComputeSectionLevels_inactiveWithoutOutline() { + TaggingTokenCluster methods = sectionCluster("2 Methods"); + TaggingTokenCluster data = sectionCluster("2.1 Data"); + + SectionLevelInfo info = TEIFormatter.computeSectionLevels(List.of(methods, data), null); + + assertThat(info.active, is(false)); + } + + @Test + public void testComputeSectionLevels_mainIsLevel1SubIsLevel2() { + TaggingTokenCluster methods = sectionCluster("2 Methods"); + TaggingTokenCluster data = sectionCluster("2.1 Data"); + TaggingTokenCluster para = paragraphCluster("Body of the data section."); + + SectionLevelInfo info = TEIFormatter.computeSectionLevels( + List.of(methods, data, para), + buildOutline()); + + assertThat(info.active, is(true)); + // main section -> level 1, its sub-section -> level 2 + assertThat(info.levels.get(methods), is(1)); + assertThat(info.levels.get(data), is(2)); + } + + @Test + public void testComputeSectionLevels_missedParentDoesNotAffectOtherHeads() { + // GROBID missed the "3 Results" heading: "3.1 Findings" appears right after "2 Methods". + // Each head is levelled independently from the outline, so the missing heading affects + // nothing else: Methods stays level 1, Findings gets its own outline level (3.1 -> level 2). + TaggingTokenCluster methods = sectionCluster("2 Methods"); + TaggingTokenCluster data = sectionCluster("2.1 Data"); + TaggingTokenCluster findings = sectionCluster("3.1 Findings"); + + SectionLevelInfo info = TEIFormatter.computeSectionLevels( + List.of(methods, data, findings), + buildOutline()); + + assertThat(info.active, is(true)); + assertThat(info.levels.get(methods), is(1)); + assertThat(info.levels.get(data), is(2)); + assertThat(info.levels.get(findings), is(2)); + } + + @Test + public void testFindOutlineDepthForHead_matchesWhenOutlineOmitsSectionNumber() { + // Many outlines store the plain title while the detected head carries a number. A short + // title like "Results" scores below the similarity threshold against "2. Results", so the + // number-stripped fallback is what makes the head matchable at all. + DocumentNode root = new DocumentNode(); + DocumentNode results = new DocumentNode("Results", null); + DocumentNode sub = new DocumentNode("Design of Primers for U. virens Detection", null); + root.addChild(results); + results.addChild(sub); + + // root is depth 0; results depth 1; sub depth 2 + assertThat(TEIFormatter.findOutlineDepthForHead(root, "2. Results"), is(1)); + assertThat( + TEIFormatter.findOutlineDepthForHead(root, "2.1. Design of Primers for U. virens Detection"), + is(2)); + assertThat(TEIFormatter.findOutlineDepthForHead(root, "Nonexistent section"), is(-1)); + } + + @Test + public void testComputeSectionLevels_unmatchedHeadNotLevelled() { + TaggingTokenCluster methods = sectionCluster("2 Methods"); + TaggingTokenCluster data = sectionCluster("2.1 Data"); + // a head that does not exist in the outline: must not be levelled (omit when unknown) + TaggingTokenCluster unknown = sectionCluster("Appendix Z Nonexistent"); + + SectionLevelInfo info = TEIFormatter.computeSectionLevels( + List.of(methods, data, unknown), + buildOutline()); + + assertThat(info.active, is(true)); + assertThat(info.levels.containsKey(unknown), is(false)); + } + } diff --git a/grobid-core/src/test/java/org/grobid/core/sax/PDFALTOOutlineSaxHandlerTest.java b/grobid-core/src/test/java/org/grobid/core/sax/PDFALTOOutlineSaxHandlerTest.java deleted file mode 100644 index 29def5dfbb..0000000000 --- a/grobid-core/src/test/java/org/grobid/core/sax/PDFALTOOutlineSaxHandlerTest.java +++ /dev/null @@ -1,135 +0,0 @@ -/* - * Copyright 2008-2026 GROBID contributors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.grobid.core.sax; - -import static org.easymock.EasyMock.createMock; -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.CoreMatchers.nullValue; -import static org.hamcrest.collection.IsCollectionWithSize.hasSize; -import static org.junit.Assert.assertThat; -import static org.junit.Assert.assertTrue; - -import java.io.InputStream; - -import javax.xml.parsers.SAXParser; -import javax.xml.parsers.SAXParserFactory; - -import org.junit.Before; -import org.junit.Test; - -import org.grobid.core.document.Document; -import org.grobid.core.document.DocumentNode; -import org.grobid.core.document.DocumentSource; - -public class PDFALTOOutlineSaxHandlerTest { - SAXParserFactory spf = SAXParserFactory.newInstance(); - - PDFALTOOutlineSaxHandler target; - DocumentSource mockDocumentSource; - Document document; - - @Before - public void setUp() throws Exception { - - mockDocumentSource = createMock(DocumentSource.class); - - document = Document.createFromText(""); - target = new PDFALTOOutlineSaxHandler(document); - } - - @Test - public void testParsing_pdf2XMLOutline_ShouldWork() throws Exception { - InputStream is = this.getClass().getResourceAsStream("pdfalto.xml_outline.xml"); - - SAXParser p = spf.newSAXParser(); - p.parse(is, target); - - DocumentNode root = target.getRootNode(); - assertTrue(root.getChildren().size() > 0); - assertThat(root.getChildren(), hasSize(9)); - assertThat(root.getChildren().get(0).getLabel(), is("Abstract")); - assertThat(root.getChildren().get(0).getChildren(), is(nullValue())); - assertThat(root.getChildren().get(0).getBoundingBox().getPage(), is(1)); - // - // assertThat(root.getChildren().get(0).getBoundingBox().getY(), is(0.0)); - // assertThat(root.getChildren().get(0).getBoundingBox().getHeight(), is(-1.0)); - // assertThat(root.getChildren().get(0).getBoundingBox().getX(), is(0.0)); - // assertThat(root.getChildren().get(0).getBoundingBox().getWidth(), is(0.0)); - } - - @Test - public void testParsing_pdf2XMLOutline_errorcase_ShouldWork() throws Exception { - InputStream is = this.getClass().getResourceAsStream("test_outline.xml"); - - SAXParser p = spf.newSAXParser(); - p.parse(is, target); - - DocumentNode root = target.getRootNode(); - assertThat(root.getChildren(), hasSize(5)); - - assertThat(root.getChildren().get(0).getLabel(), is("A Identification")); - assertThat(root.getChildren().get(0).getChildren(), is(nullValue())); - // - assertThat(root.getChildren().get(0).getBoundingBox().getPage(), is(2)); - // assertThat(root.getChildren().get(0).getBoundingBox().getY(), is(71.000)); - // assertThat(root.getChildren().get(0).getBoundingBox().getHeight(), is(0.0)); - // assertThat(root.getChildren().get(0).getBoundingBox().getX(), is(68.000)); - // assertThat(root.getChildren().get(0).getBoundingBox().getWidth(), is(0.0)); - - assertThat(root.getChildren().get(1).getLabel(), is("B Résumé consolidé public.")); - assertThat(root.getChildren().get(1).getChildren(), hasSize(1)); - // - assertThat(root.getChildren().get(1).getBoundingBox().getPage(), is(2)); - // assertThat(root.getChildren().get(1).getBoundingBox().getY(), is(377.000)); - // assertThat(root.getChildren().get(1).getBoundingBox().getHeight(), is(0.0)); - // assertThat(root.getChildren().get(1).getBoundingBox().getX(), is(68.000)); - // assertThat(root.getChildren().get(1).getBoundingBox().getWidth(), is(0.0)); - - assertThat(root.getChildren().get(1).getChildren(), hasSize(1)); - assertThat( - root.getChildren().get(1).getChildren().get(0).getLabel(), - is("B.1 Résumé consolidé public en français")); - // - assertThat(root.getChildren().get(1).getChildren().get(0).getBoundingBox().getPage(), is(2)); - // assertThat(root.getChildren().get(1).getChildren().get(0).getBoundingBox().getY(), is(412.000)); - // assertThat(root.getChildren().get(1).getChildren().get(0).getBoundingBox().getHeight(), is(0.0)); - // assertThat(root.getChildren().get(1).getChildren().get(0).getBoundingBox().getX(), is(68.000)); - // assertThat(root.getChildren().get(1).getChildren().get(0).getBoundingBox().getWidth(), is(0.0)); - - assertThat(root.getChildren().get(2).getLabel(), is("C Mémoire scientifique en français")); - assertThat(root.getChildren().get(2).getChildren(), hasSize(6)); - assertThat( - root.getChildren().get(2).getChildren().get(2).getLabel(), - is("C.3 Approche scientifique et technique")); // codespell:ignore approche - assertThat(root.getChildren().get(3).getLabel(), is("D Liste des livrables")); - assertThat(root.getChildren().get(3).getChildren(), is(nullValue())); - assertThat(root.getChildren().get(4).getLabel(), is("E Impact du projet")); // codespell:ignore projet - assertThat(root.getChildren().get(4).getChildren(), hasSize(4)); - assertThat( - root.getChildren().get(4).getChildren().get(1).getLabel(), - is("E.2 Liste des publications et communications")); - assertThat( - root.getChildren().get(4).getChildren().get(2).getLabel(), - is("E.3 Liste des autres valorisations scientifiques")); - // - assertThat(root.getChildren().get(4).getChildren().get(2).getBoundingBox().getPage(), is(1)); - // assertThat(root.getChildren().get(4).getChildren().get(2).getBoundingBox().getY(), is(170.000)); - // assertThat(root.getChildren().get(4).getChildren().get(2).getBoundingBox().getHeight(), is(0.0)); - // assertThat(root.getChildren().get(4).getChildren().get(2).getBoundingBox().getX(), is(68.000)); - // assertThat(root.getChildren().get(4).getChildren().get(2).getBoundingBox().getWidth(), is(0.0)); - } - -} diff --git a/grobid-core/src/test/kotlin/org/grobid/core/document/DocumentNodeTest.kt b/grobid-core/src/test/kotlin/org/grobid/core/document/DocumentNodeTest.kt new file mode 100644 index 0000000000..54fd223f7d --- /dev/null +++ b/grobid-core/src/test/kotlin/org/grobid/core/document/DocumentNodeTest.kt @@ -0,0 +1,100 @@ +/* + * Copyright 2008-2026 GROBID contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.grobid.core.document + +import org.junit.Test +import kotlin.test.assertEquals + +class DocumentNodeTest { + @Test + fun testFindNodeDepth() { + // Build a simple tree: + // root + // ├── child1 + // │ └── grandchild1 + // └── child2 + val root = DocumentNode("1| Introduction", "0") + val child1 = DocumentNode("2| Crystal structure", null) + val child2 = DocumentNode("child2", null) + val grandchild1 = DocumentNode("grandchild1", null) + root.addChild(child1) + root.addChild(child2) + child1.addChild(grandchild1) + + // Exact match + assertEquals(0, DocumentNode.findNodeDepth(root, "1| Introduction", 0)) + assertEquals(1, DocumentNode.findNodeDepth(root, "2| Crystal structure", 0)) + assertEquals(1, DocumentNode.findNodeDepth(root, "child2", 0)) + assertEquals(2, DocumentNode.findNodeDepth(root, "grandchild1", 0)) + + // Soft match (case-insensitive, partial, etc.) + assertEquals(1, DocumentNode.findNodeDepth(root, "2| Crystal structure", 0)) + assertEquals(1, DocumentNode.findNodeDepth(root, "Crystal structure", 0)) + assertEquals( + -1, + DocumentNode.findNodeDepth( + root, + "2.3 | Crystal structure determination of\n" + + "4a, 5a, 5b, 6a, and 6b", + 0, + ), + ) + assertEquals(2, DocumentNode.findNodeDepth(root, "grandchild", 0)) + + // Not found + assertEquals(-1, DocumentNode.findNodeDepth(root, "nonexistent", 0)) + } + + @Test + fun testFindNodeDepth_normalizesOutlineArtifacts() { + // Outline labels from pdfalto carry non-breaking spaces, soft hyphens and '|' number + // separators; the section head produced from LayoutTokens has none of these. + val root = DocumentNode("root", "0") + val nbspNode = DocumentNode("4.1 Extraction\u00A0and\u00A0isolation", null) + val softHyphenNode = DocumentNode("2.2.1|Synthesis of tert-\u00ADButoxy acetic acid", null) + root.addChild(nbspNode) + root.addChild(softHyphenNode) + + assertEquals(1, DocumentNode.findNodeDepth(root, "4.1 Extraction and isolation", 0)) + assertEquals(1, DocumentNode.findNodeDepth(root, "2.2.1 Synthesis of tert-Butoxy acetic acid", 0)) + } + + @Test + fun testFindNode() { + // root -> title -> { methods -> data , results -> findings } + val root = DocumentNode("root", "0") + val title = DocumentNode("Some Article Title", null) + val methods = DocumentNode("2 Methods", null) + val data = DocumentNode("2.1 Data", null) + val results = DocumentNode("3 Results", null) + val findings = DocumentNode("3.1 Findings", null) + root.addChild(title) + title.addChild(methods) + methods.addChild(data) + title.addChild(results) + results.addChild(findings) + + // findNode returns the matched node itself + assertEquals(methods, DocumentNode.findNode(root, "2 Methods")) + assertEquals(data, DocumentNode.findNode(root, "2.1 Data")) + assertEquals(findings, DocumentNode.findNode(root, "3.1 Findings")) + assertEquals(null, DocumentNode.findNode(root, "Nonexistent section")) + + // findNodeDepth measures from the passed root (depth 0): title=1, section=2, sub=3 + assertEquals(2, DocumentNode.findNodeDepth(root, "2 Methods", 0)) + assertEquals(3, DocumentNode.findNodeDepth(root, "2.1 Data", 0)) + } +} diff --git a/grobid-core/src/test/kotlin/org/grobid/core/sax/PDFALTOOutlineSaxHandlerTest.kt b/grobid-core/src/test/kotlin/org/grobid/core/sax/PDFALTOOutlineSaxHandlerTest.kt new file mode 100644 index 0000000000..0751478677 --- /dev/null +++ b/grobid-core/src/test/kotlin/org/grobid/core/sax/PDFALTOOutlineSaxHandlerTest.kt @@ -0,0 +1,234 @@ +/* + * Copyright 2008-2026 GROBID contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.grobid.core.sax + +import org.easymock.EasyMock +import org.grobid.core.document.Document +import org.grobid.core.document.DocumentNode +import org.grobid.core.document.DocumentNode.findNodeDepth +import org.grobid.core.document.DocumentSource +import org.hamcrest.CoreMatchers +import org.hamcrest.collection.IsCollectionWithSize +import org.junit.Assert +import org.junit.Assert.assertThat +import org.junit.Before +import org.junit.Test +import javax.xml.parsers.SAXParserFactory + +class PDFALTOOutlineSaxHandlerTest { + var spf: SAXParserFactory = SAXParserFactory.newInstance() + + var target: PDFALTOOutlineSaxHandler? = null + var mockDocumentSource: DocumentSource? = null + var document: Document? = null + + @Before + @Throws(Exception::class) + fun setUp() { + mockDocumentSource = EasyMock.createMock(DocumentSource::class.java) + + document = Document.createFromText("") + target = PDFALTOOutlineSaxHandler(document) + } + + @Test + @Throws(Exception::class) + fun testParsing_pdf2XMLOutline_ShouldWork() { + val `is` = this.javaClass.getResourceAsStream("pdfalto.xml_outline.xml") + + val p = spf.newSAXParser() + p.parse(`is`, target) + + val root = target!!.getRootNode() + Assert.assertTrue(root.getChildren().size > 0) + assertThat?>( + root.getChildren(), + IsCollectionWithSize.hasSize(9), + ) + assertThat(root.getChildren().get(0).getLabel(), CoreMatchers.`is`("Abstract")) + assertThat?>( + root.getChildren().get(0).getChildren(), + CoreMatchers.`is`( + CoreMatchers.nullValue(), + ), + ) + assertThat(root.getChildren().get(0).getBoundingBox().getPage(), CoreMatchers.`is`(1)) + // +// assertThat(root.getChildren().get(0).getBoundingBox().getY(), is(0.0)); +// assertThat(root.getChildren().get(0).getBoundingBox().getHeight(), is(-1.0)); +// assertThat(root.getChildren().get(0).getBoundingBox().getX(), is(0.0)); +// assertThat(root.getChildren().get(0).getBoundingBox().getWidth(), is(0.0)); + } + + @Test + @Throws(Exception::class) + fun testParsing_pdf2XMLOutline2_ShouldWork() { + val `is` = this.javaClass.getResourceAsStream("example1_outline.xml") + + val p = spf.newSAXParser() + p.parse(`is`, target) + + val root = target!!.rootNode + Assert.assertTrue(root.getChildren().size > 0) + // The single depth-1 node is the article title; every section hangs under it. + assertThat?>( + root.getChildren(), + IsCollectionWithSize.hasSize(1), + ) + + // Sections after the nested "4 Experimental procedures" list must reattach to the + // title node, not leak up to the root (regression guard for the parent-id stack). + val titleNode = root.getChildren().get(0) + val titleChildLabels = titleNode.getChildren().map { it!!.getLabel() } + assertThat(titleChildLabels, CoreMatchers.hasItem("Acknowledgements")) + assertThat(titleChildLabels, CoreMatchers.hasItem("References")) + + val introDepth = findNodeDepth(root, "Introduction", 0) + assertThat(introDepth, CoreMatchers.`is`(2)) + } + + @Test + @Throws(Exception::class) + fun testParsing_pdf2XMLOutline3_ShouldWork() { + val `is` = this.javaClass.getResourceAsStream("buggy_outline.xml") + + val p = spf.newSAXParser() + p.parse(`is`, target) + + val root = target!!.rootNode + Assert.assertTrue(root.getChildren().size > 0) + // The single depth-1 node is the article title; every section hangs under it. + assertThat?>( + root.getChildren(), + IsCollectionWithSize.hasSize(1), + ) + + // Labels here carry soft hyphens and a '|' number separator (e.g. "1|INTRODUCTION"), + // so this also exercises normalization in findNodeDepth. + val introDepth = findNodeDepth(root, "Introduction", 0) + assertThat(introDepth, CoreMatchers.`is`(2)) + } + + @Test + @Throws(Exception::class) + fun testParsing_pdf2XMLOutline_errorcase_ShouldWork() { + val `is` = this.javaClass.getResourceAsStream("test_outline.xml") + + val p = spf.newSAXParser() + p.parse(`is`, target) + + val root = target!!.getRootNode() + assertThat?>( + root.getChildren(), + IsCollectionWithSize.hasSize(5), + ) + + assertThat(root.getChildren().get(0).getLabel(), CoreMatchers.`is`("A Identification")) + assertThat?>( + root.getChildren().get(0).getChildren(), + CoreMatchers.`is`( + CoreMatchers.nullValue(), + ), + ) + // + assertThat(root.getChildren().get(0).getBoundingBox().getPage(), CoreMatchers.`is`(2)) + + // assertThat(root.getChildren().get(0).getBoundingBox().getY(), is(71.000)); +// assertThat(root.getChildren().get(0).getBoundingBox().getHeight(), is(0.0)); +// assertThat(root.getChildren().get(0).getBoundingBox().getX(), is(68.000)); +// assertThat(root.getChildren().get(0).getBoundingBox().getWidth(), is(0.0)); + assertThat( + root.getChildren().get(1).getLabel(), + CoreMatchers.`is`("B Résumé consolidé public."), + ) + assertThat?>( + root.getChildren().get(1).getChildren(), + IsCollectionWithSize.hasSize(1), + ) + // + assertThat(root.getChildren().get(1).getBoundingBox().getPage(), CoreMatchers.`is`(2)) + + // assertThat(root.getChildren().get(1).getBoundingBox().getY(), is(377.000)); +// assertThat(root.getChildren().get(1).getBoundingBox().getHeight(), is(0.0)); +// assertThat(root.getChildren().get(1).getBoundingBox().getX(), is(68.000)); +// assertThat(root.getChildren().get(1).getBoundingBox().getWidth(), is(0.0)); + assertThat?>( + root.getChildren().get(1).getChildren(), + IsCollectionWithSize.hasSize(1), + ) + assertThat( + root.getChildren().get(1).getChildren().get(0).getLabel(), + CoreMatchers.`is`("B.1 Résumé consolidé public en français"), + ) + // + assertThat( + root.getChildren().get(1).getChildren().get(0).getBoundingBox().getPage(), + CoreMatchers.`is`(2), + ) + + // assertThat(root.getChildren().get(1).getChildren().get(0).getBoundingBox().getY(), is(412.000)); +// assertThat(root.getChildren().get(1).getChildren().get(0).getBoundingBox().getHeight(), is(0.0)); +// assertThat(root.getChildren().get(1).getChildren().get(0).getBoundingBox().getX(), is(68.000)); +// assertThat(root.getChildren().get(1).getChildren().get(0).getBoundingBox().getWidth(), is(0.0)); + assertThat( + root.getChildren().get(2).getLabel(), + CoreMatchers.`is`("C Mémoire scientifique en français"), + ) + assertThat?>( + root.getChildren().get(2).getChildren(), + IsCollectionWithSize.hasSize(6), + ) + assertThat( + root.getChildren().get(2).getChildren().get(2).getLabel(), + CoreMatchers.`is`("C.3 Approche scientifique et technique"), + ) + assertThat( + root.getChildren().get(3).getLabel(), + CoreMatchers.`is`("D Liste des livrables"), + ) + assertThat?>( + root.getChildren().get(3).getChildren(), + CoreMatchers.`is`( + CoreMatchers.nullValue(), + ), + ) + assertThat( + root.getChildren().get(4).getLabel(), + CoreMatchers.`is`("E Impact du projet"), + ) + assertThat?>( + root.getChildren().get(4).getChildren(), + IsCollectionWithSize.hasSize(4), + ) + assertThat( + root.getChildren().get(4).getChildren().get(1).getLabel(), + CoreMatchers.`is`("E.2 Liste des publications et communications"), + ) + assertThat( + root.getChildren().get(4).getChildren().get(2).getLabel(), + CoreMatchers.`is`("E.3 Liste des autres valorisations scientifiques"), + ) + // + assertThat( + root.getChildren().get(4).getChildren().get(2).getBoundingBox().getPage(), + CoreMatchers.`is`(1), + ) + // assertThat(root.getChildren().get(4).getChildren().get(2).getBoundingBox().getY(), is(170.000)); +// assertThat(root.getChildren().get(4).getChildren().get(2).getBoundingBox().getHeight(), is(0.0)); +// assertThat(root.getChildren().get(4).getChildren().get(2).getBoundingBox().getX(), is(68.000)); +// assertThat(root.getChildren().get(4).getChildren().get(2).getBoundingBox().getWidth(), is(0.0)); + } +} diff --git a/grobid-core/src/test/resources/org/grobid/core/sax/buggy_outline.xml b/grobid-core/src/test/resources/org/grobid/core/sax/buggy_outline.xml new file mode 100644 index 0000000000..0a2a4842e5 --- /dev/null +++ b/grobid-core/src/test/resources/org/grobid/core/sax/buggy_outline.xml @@ -0,0 +1,216 @@ + + + + + Synthesis of N-­aminophalimides derived from α-­amino acids: Theoretical study to + find them as HDAC8 inhibitors by docking simulations and in vitro assays + + + + + Abstract + + + + 1|INTRODUCTION + + + + 2|EXPERIMENTAL SECTION + + + + 2.1|General procedures and materials + + + + 2.2|Chemistry + + + + 2.2.1|Synthesis of tert-­Butoxy acetic acid (2) + + + + 2.2.2|General procedure for the synthesis of compounds 3(a–­c) + + + + + 2.2.3|Synthesis of methyl 2-­(2-­(tert-­butoxy)acetamido)acetate + (3a) + + + + + 2.2.4|Synthesis of (S)-­methyl 2-­(2-­(tert-­butoxy)acetamido)-­3-­phenylpropanoate + (3b) + + + + + 2.2.5|Synthesis of (S)-­methyl 2-­(2-­(tert-­butoxy)acetamido)-­3-­(1H-­indol-­3-­yl)propanoate + (3c) + + + + + 2.2.6|General procedure for the synthesis of compounds 4(a–­c) + + + + + 2.2.7|Synthesis of 2-­(2-­(tert-­butoxy)acetamido)acetic acid + (4a) + + + + + 2.2.8|Synthesis of (S)-­2-­(2-­(tert-­butoxy)acetamido)-­3-­phenylpropanoic + acid (4b) + + + + + 2.2.9|Synthesis of (S)-­2-­(2-­(tert-­butoxy)acetamido)-­3-­(1H-­indol-­3-­yl)propanoic + acid (4c) + + + + + 2.2.10|General procedure for the synthesis of compounds + 5(a–­c) + + + + + 2.2.11|Synthesis of 2-­(tert-­butoxy)-­N-­(2-­((1,3-­dioxoisoindolin-­2-­yl)amino)-­2-­oxoethyl)acetamide + (5a) + + + + + 2.2.12|Synthesis of (S)-­2-­(2-­(tert-­butoxy)acetamido)-­N-­(1,3-­dioxoisoindolin-­2-­yl)-­3-­phenyl-­propanamide + (5b) + + + + + 2.2.13|Synthesis of (S)-­2-­(2-­(tert-­butoxy)acetamido)-­N-­(1,3-­dioxoisoindolin-­2-­yl)-­3-­(1H-­indol-­3-­yl) + propanamide (5c) + + + + + 2.2.14|General procedure for the synthesis of compounds 6a–­6c + + + + + 2.2.15|Synthesis of N-­(1,3-­dioxoisoindolin-­2-­yl)-­2-­(2-­hydroxyacetamido)acetamide + (6a) + + + + + 2.2.16|Synthesis of (S)-­N-­(1,3-­dioxoisoindolin-­2-­yl)-­2-­(2-­hydroxyacetamido)-­3-­phenylpropa-­namide + (6b) + + + + + 2.2.17|Synthesis of (S)-­N-­(1,3-­dioxoisoindolin-­2-­yl)-­2-­(2-­hydroxyacetamido)-­3-­(1H-­indol-­3-­yl)propanamide + (6c) + + + + + + + 2.3|Crystal structure determination of 4a, 5a, 5b, 6a, and 6b + + + + 2.4|Computational methodology + + + + 2.5|HDAC8 inhibition assay + + + + + + 3|RESULTS AND DISCUSSION + + + + 3.1|Chemistry + + + + 3.2|Spectroscopy + + + + 3.3|X-­ray structures + + + + + + 4|MOLECULAR DOCKING STUDY + + + + 4.1|Binding modes of compounds 3(a–­c) with HDAC8 + + + + 4.2|Binding modes of compounds 4(a–­c) with HDAC8 + + + + 4.3|Binding modes of compounds 5(a–­c) with HDAC8 + + + + 4.4|Binding modes of compounds 6(a–­c) with HDAC8 + + + + + + 5|HDAC8 INHIBITORY ASSAY + + + + 6|CONCLUSION + + + + ACKNOWLEDGMENTS + + + + FUNDING INFORMATION + + + + CONFLICT OF INTEREST STATEMENT + + + + DATA AVAILABILITY STATEMENT + + + + + + REFERENCES + + + + + + diff --git a/grobid-core/src/test/resources/org/grobid/core/sax/example1_outline.xml b/grobid-core/src/test/resources/org/grobid/core/sax/example1_outline.xml new file mode 100644 index 0000000000..3ba3a67bda --- /dev/null +++ b/grobid-core/src/test/resources/org/grobid/core/sax/example1_outline.xml @@ -0,0 +1,59 @@ + + + + + Enantiomeric diarylheptanoids from Ottelia acuminata var. acuminata and their α-glucosidase + inhibitory activity + + + + + Abstract + + + + 1 Introduction + + + + 2 Result and discussion + + + + 3 Conclusions + + + + 4 Experimental procedures + + + + 4.1 General experimental procedures + + + + 4.2 Plant material + + + + 4.3 Extraction and isolation + + + + 4.4 α-Glucosidase and PTB1B inhibitory activities detection + + + + + + Acknowledgements + + + + References + + + + + + diff --git a/grobid-home/schemas/dtd/Grobid.dtd b/grobid-home/schemas/dtd/Grobid.dtd index ea425389c8..6b9496456d 100644 --- a/grobid-home/schemas/dtd/Grobid.dtd +++ b/grobid-home/schemas/dtd/Grobid.dtd @@ -1539,7 +1539,8 @@ value CDATA #IMPLIED > %tei_att.global.attributes; %tei_att.typed.attributes; %tei_att.placement.attributes; - %tei_att.written.attributes; > + %tei_att.written.attributes; + level CDATA #IMPLIED > diff --git a/grobid-home/schemas/rng/Grobid.rng b/grobid-home/schemas/rng/Grobid.rng index 4de58d55bd..6f7d319be7 100644 --- a/grobid-home/schemas/rng/Grobid.rng +++ b/grobid-home/schemas/rng/Grobid.rng @@ -817,6 +817,24 @@ Suggested values include: 1] internal; 2] external; 3] conjecture + + + + + + + GROBID extension: bounding-box coordinates of the element in the source PDF, as a list of "page,x,y,width,height" quintuplets separated by semicolons. + + + + + + + + GROBID extension: a bare language attribute (a non-namespaced variant of xml:lang) emitted on some elements such as orgName. + + + @@ -933,12 +951,14 @@ Suggested values include: 1] internal; 2] external; 3] conjecture - - (uniform resource locator) specifies the URL from which the media concerned may be obtained. - - \S+ - - + + + (uniform resource locator) specifies the URL from which the media concerned may be obtained. Made optional: GROBID emits a <graphic> located only by @coords, with no URL. + + \S+ + + + @@ -2163,6 +2183,7 @@ Suggested values include: 1] volume (volume); 2] issue; 3] page (page); 4] line; + @@ -2594,6 +2615,8 @@ Suggested values include: 1] volume (volume); 2] issue; 3] page (page); 4] line; + + @@ -2906,6 +2929,7 @@ Suggested values include: 1] volume (volume); 2] issue; 3] page (page); 4] line; + @@ -3662,6 +3686,96 @@ Suggested values include: 1] gloss (gloss); 2] index (index); 3] instructions (i + + + GROBID extension: hierarchy level of a section head derived from the PDF outline (1 for a main section, 2 for a sub-section, and so on). + + + + + + + + + (referencing string) contains a general purpose name or referring string. GROBID uses it to mark up entities such as funders, grant numbers and grant names in running text. + + + + + + + + + + + + + + + + + + + (funding body) specifies the name of an individual, institution, or organization responsible for the funding of a project or text. + + + + + + + + + + + + + + + + + + (statement of responsibility) supplies a statement of responsibility for the intellectual content of a text, edition, recording, or series, where the specialized elements for authors, editors, etc. do not suffice or do not apply. + + + + + + + + + + + + + + + + (responsibility) contains a phrase describing the nature of a person's intellectual responsibility, or an organization's role in the production or distribution of a work. + + + + + + + + + + + + + + + (list of organizations) contains a list of elements, each of which provides information about an identifiable organization. GROBID uses it in the back matter to record funding organizations, grants and grant numbers. + + + + + + + + + + @@ -5022,6 +5136,12 @@ Suggested values include: 1] sent; 2] received; 3] transmitted; 4] redirected; 5 + + + schemaLocation + GROBID emits an xsi:schemaLocation hint on the root element. + + @@ -5187,7 +5307,9 @@ Suggested values include: 1] sent; 2] received; 3] transmitted; 4] redirected; 5 - + + @@ -5351,6 +5473,7 @@ Suggested values include: 1] sent; 2] received; 3] transmitted; 4] redirected; 5 + @@ -5361,6 +5484,7 @@ Suggested values include: 1] sent; 2] received; 3] transmitted; 4] redirected; 5 + diff --git a/grobid-home/schemas/xsd/Grobid.xsd b/grobid-home/schemas/xsd/Grobid.xsd index 017a4c43a9..88b4c2a5cd 100644 --- a/grobid-home/schemas/xsd/Grobid.xsd +++ b/grobid-home/schemas/xsd/Grobid.xsd @@ -3583,6 +3583,7 @@ Suggested values include: 1] gloss (gloss); 2] index (index); 3] instructions (i +