Use PDF outline to correct the header hierarchy

grobid-core/src/main/java/org/grobid/core/document/DocumentNode.java

@@ -18,10 +18,12 @@
 import java.util.ArrayList;
 import java.util.List;
 
+import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.builder.ToStringBuilder;
 import org.apache.commons.lang3.builder.ToStringStyle;
 
 import org.grobid.core.layout.BoundingBox;
+import org.grobid.core.utilities.TextUtilities;
 
 /**
  * Class corresponding to a node of the structure of a hierarchically organized document (i.e. for a table
@@ -222,4 +224,73 @@ public Integer getId() {
     public void setId(Integer id) {
         this.id = id;
     }
+
+    // similarity threshold above which an outline label is considered to match a section head
+    private static final double NODE_MATCH_THRESHOLD = 0.90;
+
+    /**
+     * Normalize an outline label or a section head for soft comparison. Outline labels extracted
+     * from the PDF bookmarks carry non-breaking spaces, soft hyphens, '|' number separators and
+     * multi-line indentation runs, none of which are meaningful for matching.
+     */
+    private static String normalizeForMatching(String value) {
+        if (value == null) {
+            return null;
+        }
+        String normalized = value
+                .replace('\u00A0', ' ')  // non-breaking space
+                .replace("\u00AD", "")   // soft hyphen
+                .replace('|', ' ');
+        return StringUtils.normalizeSpace(normalized);
+    }
+
+    /**
+     * Given the label of a potential node (typically a section head), find it in the hierarchy
+     * rooted at {@code rootNode} using soft string matching and return its depth.
+     *
+     * @param rootNode the node to search from
+     * @param label    the section head text to look up
+     * @param depth    the depth of {@code rootNode} in the hierarchy (0 for the outline root)
+     * @return the depth of the first matching node, or -1 if no node matches
+     */
+    public static int findNodeDepth(DocumentNode rootNode, String label, int depth) {
+        DocumentNode node = findNode(rootNode, label);
+        if (node == null) {
+            return -1;
+        }
+        int d = depth;
+        DocumentNode cursor = node;
+        while (cursor != rootNode && cursor.getFather() != null) {
+            d++;
+            cursor = cursor.getFather();
+        }
+        return d;
+    }
+
+    /**
+     * Find the first node in the hierarchy rooted at {@code rootNode} whose label softly matches
+     * {@code label} (depth-first). Returns the matching node itself, or null if none matches.
+     */
+    public static DocumentNode findNode(DocumentNode rootNode, String label) {
+        if (rootNode == null || label == null) {
+            return null;
+        }
+        String normalizedLabel = normalizeForMatching(label);
+        String nodeLabel = normalizeForMatching(rootNode.getLabel());
+        if (StringUtils.isNotBlank(nodeLabel)) {
+            double score = TextUtilities.getRatcliffObershelpSimilarity(nodeLabel, normalizedLabel, false);
+            if (score >= NODE_MATCH_THRESHOLD) {
+                return rootNode;
+            }
+        }
+        if (rootNode.getChildren() != null) {
+            for (DocumentNode child : rootNode.getChildren()) {
+                DocumentNode found = findNode(child, label);
+                if (found != null) {
+                    return found;
+                }
+            }
+        }
+        return null;
+    }
 }

grobid-core/src/main/java/org/grobid/core/document/TEIFormatter.java

@@ -1609,6 +1609,84 @@ public StringBuilder toTEIAnnex(
         return buffer;
     }
 
+    /**
+     * Per-head hierarchy level derived from the PDF outline: 1 for a main section, 2 for a
+     * sub-section, and so on. Keyed by cluster identity (not head text, which can repeat across a
+     * document). {@code active} is true when the outline gives enough signal to annotate this piece.
+     */
+    static class SectionLevelInfo {
+        final Map<TaggingTokenCluster, Integer> levels;
+        final boolean active;
+
+        SectionLevelInfo(Map<TaggingTokenCluster, Integer> levels, boolean active) {
+            this.levels = levels;
+            this.active = active;
+        }
+    }
+
+    /**
+     * Compute, for each SECTION head that can be located in the document outline, its hierarchy level:
+     * 1 for the shallowest matched section, +1 for each outline level below it. The outline is the
+     * only signal that reveals when a section returns to a higher level (the "ascent"), which cannot
+     * be inferred from the text alone. Activated only when the outline exists, the piece has at least
+     * one pair of adjacent section heads (evidence of hierarchy), and at least one head matched.
+     */
+    static SectionLevelInfo computeSectionLevels(List<TaggingTokenCluster> clusters, DocumentNode outlineRoot) {
+        Map<TaggingTokenCluster, Integer> depths = new IdentityHashMap<>();
+        int minDepth = Integer.MAX_VALUE;
+
+        boolean hasAdjacentSectionHeads = false;
+        TaggingLabel previousLabel = null;
+        for (TaggingTokenCluster cluster : clusters) {
+            if (cluster == null) {
+                continue;
+            }
+            TaggingLabel clusterLabel = cluster.getTaggingLabel();
+            if (TaggingLabels.SECTION.equals(clusterLabel) && TaggingLabels.SECTION.equals(previousLabel)) {
+                hasAdjacentSectionHeads = true;
+            }
+            previousLabel = clusterLabel;
+        }
+
+        if (hasAdjacentSectionHeads && outlineRoot != null) {
+            for (TaggingTokenCluster cluster : clusters) {
+                if (cluster == null || !TaggingLabels.SECTION.equals(cluster.getTaggingLabel())) {
+                    continue;
+                }
+                String clusterContent = LayoutTokensUtil.normalizeDehyphenizeText(cluster.concatTokens());
+                int depth = findOutlineDepthForHead(outlineRoot, clusterContent);
+                if (depth > 0) {
+                    depths.put(cluster, depth);
+                    minDepth = Math.min(minDepth, depth);
+                }
+            }
+        }
+
+        Map<TaggingTokenCluster, Integer> levels = new IdentityHashMap<>();
+        for (Map.Entry<TaggingTokenCluster, Integer> entry : depths.entrySet()) {
+            levels.put(entry.getKey(), entry.getValue() - minDepth + 1);
+        }
+        boolean active = hasAdjacentSectionHeads && !levels.isEmpty();
+        return new SectionLevelInfo(levels, active);
+    }
+
+    /**
+     * Depth of the outline node matching a section head, or -1 if none. Tries the head text as
+     * labelled first (keeping the section number to disambiguate same-titled sections), then retries
+     * with the number stripped, since many outlines store the bare title ("Results") while the head
+     * carries a number ("2. Results") that pushes a short title below the similarity threshold.
+     */
+    static int findOutlineDepthForHead(DocumentNode outlineRoot, String headText) {
+        int depth = DocumentNode.findNodeDepth(outlineRoot, headText, 0);
+        if (depth < 0) {
+            org.grobid.core.utilities.Pair<String, String> numb = getSectionNumber(headText);
+            if (numb != null && StringUtils.isNotBlank(numb.a)) {
+                depth = DocumentNode.findNodeDepth(outlineRoot, numb.a, 0);
+            }
+        }
+        return depth;
+    }
+
     public StringBuilder toTEITextPiece(
             StringBuilder buffer,
             String result,
@@ -1638,6 +1716,11 @@ public StringBuilder toTEITextPiece(
 
         List<Element> divResults = new ArrayList<>();
 
+        // When the PDF provides an outline (table of content), annotate each section head with its
+        // hierarchy level (1 = main section, 2 = sub-section, ...). The div structure is left flat and
+        // identical to the outline-less output; only a @level attribute is added on matched heads.
+        SectionLevelInfo sectionLevelInfo = computeSectionLevels(clusters, doc.getOutlineRoot());
+
         Element curDiv = teiElement("div");
         if (config.isGenerateTeiIds()) {
             String divID = KeyGen.getKey().substring(0, 7);
@@ -1668,6 +1751,12 @@ public StringBuilder toTEITextPiece(
                     head.appendChild(clusterContent);
                 }
 
+                // hierarchy level from the PDF outline, when this head could be located in it
+                Integer level = sectionLevelInfo.levels.get(cluster);
+                if (level != null) {
+                    head.addAttribute(new Attribute("level", String.valueOf(level)));
+                }
+
                 if (config.isGenerateTeiIds()) {
                     String divID = KeyGen.getKey().substring(0, 7);
                     addXmlId(head, "_" + divID);
@@ -2381,7 +2470,7 @@ private List<GraphicObject> getGraphicObject(List<GraphicObject> graphicObjects,
         return result;
     }
 
-    private org.grobid.core.utilities.Pair<String, String> getSectionNumber(String text) {
+    private static org.grobid.core.utilities.Pair<String, String> getSectionNumber(String text) {
         Matcher m1 = BasicStructureBuilder.headerNumbering1.matcher(text);
         Matcher m2 = BasicStructureBuilder.headerNumbering2.matcher(text);
         Matcher m3 = BasicStructureBuilder.headerNumbering3.matcher(text);

grobid-core/src/main/java/org/grobid/core/engines/FullTextParser.java

@@ -99,7 +99,7 @@ public Document processing(File inputPdf,
                                GrobidAnalysisConfig config) throws Exception {
         DocumentSource documentSource =
             DocumentSource.fromPdf(inputPdf, config.getStartPage(), config.getEndPage(),
-                config.getPdfAssetPath() != null, true, false);
+                config.getPdfAssetPath() != null, true, true);
         documentSource.setMD5(md5Str);
         return processing(documentSource, flavor, config);
     }

grobid-core/src/main/java/org/grobid/core/sax/PDFALTOOutlineSaxHandler.java

@@ -44,7 +44,9 @@ public class PDFALTOOutlineSaxHandler extends DefaultHandler {
 
     private int currentLevel = -1;
     private int currentId = -1;
-    private int currentParentId = -1;
+    // parent id of the TOCITEMLIST currently open, one entry per nesting level;
+    // -1 when the list has no idItemParent (root list)
+    private Deque<Integer> currentParentIdStack = new ArrayDeque<>();
 
     private Map<Integer, DocumentNode> nodes = null;
 
@@ -80,7 +82,12 @@ public void endElement(
             box = null;
             label = null;
         } else if (qName.equals("TOCITEMLIST")) {
-            currentParentId = -1;
+            if (!currentParentIdStack.isEmpty()) {
+                currentParentIdStack.pop();
+            } else {
+                LOGGER.warn(
+                        "TOCITEMLIST end encountered with empty parent stack. Possible malformed outline structure.");
+            }
         } else if (qName.equals("LINK")) {
             // in case of nested item, we need to assign the box right away or we will lose it.
             if (box != null) {
@@ -123,7 +130,8 @@ public void startElement(
             }
             currentNode.setId(currentId);
             nodes.put(currentId, currentNode);
-            if (currentParentId != -1) {
+            Integer currentParentId = currentParentIdStack.peek();
+            if (currentParentId != null && currentParentId != -1) {
                 DocumentNode father = nodes.get(currentParentId);
                 if (father == null)
                     LOGGER.warn("Father not yet encountered! id is " + currentParentId);
@@ -141,6 +149,9 @@ public void startElement(
             // we only consider annotation with attribute @subtype of value "Link"
             int length = atts.getLength();
 
+            // the root TOCITEMLIST has no idItemParent attribute
+            int parentId = -1;
+
             // Process attributes
             for (int i = 0; i < length; i++) {
                 // Get names and values for each attribute
@@ -157,14 +168,16 @@ public void startElement(
                         }
                     } else if (name.equals("idItemParent")) {
                         try {
-                            currentParentId = Integer.parseInt(value);
+                            parentId = Integer.parseInt(value);
                         } catch (Exception e) {
                             LOGGER.warn("Invalid parent id string (should be an integer): " + value);
-                            currentParentId = -1;
+                            parentId = -1;
                         }
                     }
                 }
             }
+            // always push exactly one entry per TOCITEMLIST so that start/end events stay balanced
+            currentParentIdStack.push(parentId);
         } else if (qName.equals("LINK")) {
             int length = atts.getLength();