Support for documents in humanities, Law, and pre-1900 layouts (footnotes-driven)

Dockerfile.evaluation

@@ -87,6 +87,8 @@ COPY --from=builder /opt/grobid .
 RUN python3 -m pip install pip --upgrade --no-cache-dir
 
 # install DeLFT
+# pin lmdb to avoid breaking change in 2.0.0 (duplicate env open rejection)
+RUN pip3 install --no-cache-dir "lmdb<2.0.0"
 RUN pip3 install --no-cache-dir delft==0.4.6
 
 # link the data directory to /data

build.gradle

@@ -574,23 +574,26 @@ project(":grobid-trainer") {
         "train_table"                  : "org.grobid.trainer.TableTrainer",
         "train_citation"               : "org.grobid.trainer.CitationTrainer",
         "train_date"                   : "org.grobid.trainer.DateTrainer",
-        "train_reference_segmentation" : "org.grobid.trainer.ReferenceSegmenterTrainer",
         "train_patent_citation"        : "org.grobid.trainer.PatentParserTrainer",
         "train_funding_acknowledgement": "org.grobid.trainer.FundingAcknowledgementTrainer"
     ]
 
     def complexTrainerTasks = [
-        "train_header"                        : ["org.grobid.trainer.HeaderTrainer", ""],
-        "train_header_article_light"          : ["org.grobid.trainer.HeaderTrainer", "article/light"],
-        "train_header_article_light_ref"      : ["org.grobid.trainer.HeaderTrainer", "article/light-ref"],
-        "train_header_ietf"                   : ["org.grobid.trainer.HeaderTrainer", "sdo/ietf"],
-        "train_segmentation"                  : ["org.grobid.trainer.SegmentationTrainer", ""],
-        "train_segmentation_article_light"    : ["org.grobid.trainer.SegmentationTrainer", "article/light"],
-        "train_segmentation_article_light_ref": ["org.grobid.trainer.SegmentationTrainer", "article/light-ref"],
-        "train_segmentation_ietf"             : ["org.grobid.trainer.SegmentationTrainer", "sdo/ietf"],
-        "train_fulltext"                      : ["org.grobid.trainer.FulltextTrainer", ""],
-        "train_fulltext_article_light"        : ["org.grobid.trainer.FulltextTrainer", "article/light"],
-        "train_fulltext_article_light_ref"    : ["org.grobid.trainer.FulltextTrainer", "article/light-ref"],
+        "train_header"                                          : ["org.grobid.trainer.HeaderTrainer", ""],
+        "train_header_article_light"                            : ["org.grobid.trainer.HeaderTrainer", "article/light"],
+        "train_header_article_light_ref"                        : ["org.grobid.trainer.HeaderTrainer", "article/light-ref"],
+        "train_header_ietf"                                     : ["org.grobid.trainer.HeaderTrainer", "sdo/ietf"],
+        "train_segmentation"                                    : ["org.grobid.trainer.SegmentationTrainer", ""],
+        "train_segmentation_article_light"                      : ["org.grobid.trainer.SegmentationTrainer", "article/light"],
+        "train_segmentation_article_light_ref"                  : ["org.grobid.trainer.SegmentationTrainer", "article/light-ref"],
+        "train_segmentation_article_dh_law_footnotes"           : ["org.grobid.trainer.SegmentationTrainer", "article/dh-law-footnotes"],
+        "train_segmentation_article_dh_law_footnotes_token"     : ["org.grobid.trainer.SegmentationTrainer", "article/dh-law-footnotes-token"],
+        "train_reference_segmenter"                             : ["org.grobid.trainer.ReferenceSegmenterTrainer", ""],
+        "train_reference_segmenter_article_dh_law_footnotes"    : ["org.grobid.trainer.ReferenceSegmenterTrainer", "article/dh-law-footnotes"],
+        "train_segmentation_ietf"                               : ["org.grobid.trainer.SegmentationTrainer", "sdo/ietf"],
+        "train_fulltext"                                        : ["org.grobid.trainer.FulltextTrainer", ""],
+        "train_fulltext_article_light"                          : ["org.grobid.trainer.FulltextTrainer", "article/light"],
+        "train_fulltext_article_light_ref"                      : ["org.grobid.trainer.FulltextTrainer", "article/light-ref"],
     ]
 
     def libraries = ""

doc/Grobid-service.md

@@ -1038,6 +1038,19 @@ curl --location 'http://localhost:8070/api/createTraining' \
 
 The zip file will contain the training data as described in the [general principles](training/General-principles.md) of GROBID training data.
 
+### createTrainingBlank
+
+Generate blank training data (raw features + empty TEI template) from a PDF without requiring a trained model. This is useful for creating training data from scratch.
+
+```bash
+curl --location 'http://localhost:8070/api/createTrainingBlank' \
+--form 'input=@"PATH_DOCUMENT"' \
+--form 'flavor=article/dh-law-footnotes-token' \
+--output output_name.zip
+```
+
+The `flavor` parameter is optional. When set, the generated raw feature files will use the feature extraction strategy of that flavor (e.g., token-level features for `article/dh-law-footnotes-token`).
+
 ## Parallel mode
 
 The Grobid REST API provides a very efficient way to use the library out of the box, because the service exploits multithreading.

doc/benchmarks/flavors/article_dh_law/readme.md

@@ -0,0 +1,66 @@
+# Benchmarks
+
+## Iterations batch 1,2,3 
+Date: 2025-10-25
+
+### List of files used for training: 
+
+10.1093$1$gerhis$1$ghaf028.training.segmentation.tei.xml
+10.3249$1$1868-1581-2-2-clark.training.segmentation.tei.xml
+10.5354$1$rti.v1i1.21339.training.segmentation.tei.xml
+10.5354$1$rti.v1i2.25648.training.segmentation.tei.xml
+10.5771$1$2699-1284-2024-3-149.training.segmentation.tei.xml
+10.5771__2699-1284-2020-1-83.training.segmentation.tei.xml
+10.6092__issn.2531-6133__10402.training.segmentation.tei.xml
+10.12759$1$hsr.6.1981.3.3-17.training.segmentation.tei.xml
+10.12775$1$clr.2013.004.training.segmentation.tei.xml
+10.12775$1$clr.2013.008.training.segmentation.tei.xml
+10.14276$1$2384-8901$1$448.training.segmentation.tei.xml
+10.16995$1$lefou.9.training.segmentation.tei.xml
+10.19164__ijple.v6i1.1295.training.segmentation.tei.xml
+10.19195$1$0524-4544.337.9.training.segmentation.tei.xml
+10.25364%2F01.10%3A2023.2.1.training.segmentation.tei.xml
+10.25364%2F01.11%3A2024.1.5.training.segmentation.tei.xml
+10.26321$1$a.facchinetti.01.2025.05.training.segmentation.tei.xml
+10.26321$1$s.marino.01.2025.06.training.segmentation.tei.xml
+10.32361$1$2025170222092.training.segmentation.tei.xml
+10.34767$1$dp.2024.02.02.training.segmentation.tei.xml
+1296-Article Text-4476-1-10-20221017.training.segmentation.tei.xml
+
+### List of files used for evaluation: 
+
+10.1093$1$gerhis$1$ghae045.training.segmentation.tei.xml
+10.3249$1$1868-1581-1-2-gutbrod.training.segmentation.tei.xml
+10.5771__2699-1284-2020-1-16.training.segmentation.tei.xml
+10.6092__issn.2531-6133__6356.training.segmentation.tei.xml
+10.12759$1$hsr.3.1978.1.3-10.training.segmentation.tei.xml
+10.12775$1$clr.2013.002.training.segmentation.tei.xml
+10.14276$1$2384-8901$1$443.training.segmentation.tei.xml
+10.19164__ijple.v6i1.1293.training.segmentation.tei.xml
+10.19195$1$0524-4544.337.2.training.segmentation.tei.xml
+10.32361$1$201810011903.training.segmentation.tei.xml
+1294-Article Text-4464-1-10-20221017.training.segmentation.tei.xml
+
+### Results 
+
+===== Field-level results =====
+
+label                accuracy     precision    recall       f1           support
+
+<body>               91.57        79.45        82.86        81.12        210    
+<cover>              99.79        0            0            0            0      
+<footnote>           98.54        64.29        50           56.25        18     
+<header>             96.98        36.84        29.17        32.56        24     
+<headnote>           91.16        59.86        75.22        66.67        113    
+<page>               96.15        85.71        96.88        90.95        192    
+<references>         91.05        76.06        82.23        79.02        197    
+<toc>                99.79        0            0            0            2      
+
+all (micro avg.)     95.03        75.61        82.41        78.86        756    
+all (macro avg.)     95.03        57.46        59.48        58.08        756    
+
+===== Instance-level results =====
+
+Total expected instances:   11
+Correct instances:          0
+Instance-level recall:      0

grobid-core/src/main/java/org/grobid/core/GrobidModels.java

@@ -97,6 +97,8 @@ public enum Flavor {
         BLANK("blank"),
         ARTICLE_LIGHT("article/light"),
         ARTICLE_LIGHT_WITH_REFERENCES("article/light-ref"),
+        ARTICLE_DH_LAW_FOOTNOTES("article/dh-law-footnotes"),
+        ARTICLE_DH_LAW_FOOTNOTES_TOKEN("article/dh-law-footnotes-token"),
         _3GPP("sdo/3gpp"),
         IETF("sdo/ietf");
 

grobid-core/src/main/java/org/grobid/core/data/BiblioItem.java

@@ -64,6 +64,9 @@ public class BiblioItem {
     private String teiId;
     //TODO: keep in sync with teiId - now teiId is generated in many different places
     private Integer ordinal;
+    // optional grouping label emitted as @n on the biblStruct (e.g. the footnote number shared by
+    // several references belonging to the same footnote in the dh-law-footnotes flavour)
+    private String teiGroupLabel;
     private List<BoundingBox> coordinates = null;
 
     // map of labels (e.g. <title> or <abstract>) to LayoutToken
@@ -794,6 +797,14 @@ public String getReference() {
         return reference;
     }
 
+    public String getTeiGroupLabel() {
+        return teiGroupLabel;
+    }
+
+    public void setTeiGroupLabel(String teiGroupLabel) {
+        this.teiGroupLabel = teiGroupLabel;
+    }
+
     public String getCopyright() {
         return copyright;
     }
@@ -2300,20 +2311,27 @@ public String toTEI(int n, int indent, GrobidAnalysisConfig config) {
                 tei.append(" source=\"" + getConsolidationService() + "\"");
             tei.append(" ");
 
+            // optional grouping attribute: references sharing the same @n belong to the same footnote
+            // (dh-law-footnotes flavour). Additive only - the xml:id "b{n}" is left untouched so that
+            // body callouts <ref target="#bN"> keep resolving.
+            String groupLabelAttr = StringUtils.isBlank(teiGroupLabel)
+                    ? ""
+                    : " n=\"" + TextUtilities.HTMLEncode(teiGroupLabel) + "\"";
+
             if (!StringUtils.isEmpty(language)) {
                 if (n == -1) {
                     tei.append("xml:lang=\"" + language + ">\n");
                 } else {
                     teiId = "b" + n;
-                    tei.append("xml:lang=\"" + language + "\" xml:id=\"" + teiId + "\">\n");
+                    tei.append("xml:lang=\"" + language + "\" xml:id=\"" + teiId + "\"" + groupLabelAttr + ">\n");
                 }
                 // TBD: we need to ensure that the language is normalized following xml lang attributes !
             } else {
                 if (n == -1) {
                     tei.append(">\n");
                 } else {
                     teiId = "b" + n;
-                    tei.append("xml:id=\"" + teiId + "\">\n");
+                    tei.append("xml:id=\"" + teiId + "\"" + groupLabelAttr + ">\n");
                 }
             }
 

grobid-core/src/main/java/org/grobid/core/document/BasicStructureBuilder.java

@@ -303,6 +303,97 @@ static public Document generalResultSegmentation(
         return doc;
     }
 
+    /**
+     * Token-level variant of generalResultSegmentation. Instead of receiving one label per
+     * document-line, this method receives one label per token and maps them directly to
+     * DocumentPiece boundaries.
+     */
+    static public Document generalResultSegmentationTokenLevel(
+            Document doc,
+            String labeledResult,
+            List<LayoutToken> documentTokens) {
+        List<Pair<String, String>> labeledTokens = GenericTaggerUtils.getTokensAndLabels(labeledResult);
+
+        SortedSetMultimap<String, DocumentPiece> labeledBlocks = TreeMultimap.create();
+        doc.setLabeledBlocks(labeledBlocks);
+
+        List<Block> docBlocks = doc.getBlocks();
+        if (docBlocks == null || docBlocks.isEmpty() || documentTokens == null || documentTokens.isEmpty()) {
+            return doc;
+        }
+
+        // build a mapping from tokenDocPos -> blockIndex
+        int[] tokenToBlock = new int[documentTokens.size()];
+        for (int bi = 0; bi < docBlocks.size(); bi++) {
+            Block block = docBlocks.get(bi);
+            int start = block.getStartToken();
+            int end = block.getEndToken();
+            if (start == -1 || end == -1)
+                continue;
+            for (int t = start; t <= end && t < documentTokens.size(); t++) {
+                tokenToBlock[t] = bi;
+            }
+        }
+
+        int docTokenPos = 0; // position in documentTokens
+        String lastPlainLabel = null;
+        int segmentStartPos = -1; // tokenDocPos where the current segment starts
+        int segmentEndPos = -1;   // tokenDocPos where the current segment ends
+
+        String ignoredLabel = "@IGNORED_LABEL@";
+        for (Pair<String, String> labeledTokenPair : Iterables.concat(
+                labeledTokens,
+                Collections.singleton(Pair.of("IgnoredToken", ignoredLabel)))) {
+            if (labeledTokenPair == null) {
+                continue;
+            }
+
+            String curLabel = labeledTokenPair.getRight();
+            String curPlainLabel = GenericTaggerUtils.getPlainLabel(curLabel);
+            String labeledTokenText = labeledTokenPair.getLeft();
+
+            // advance docTokenPos to find the matching document token (skip whitespace/newlines)
+            while (docTokenPos < documentTokens.size()) {
+                String docTokText = documentTokens.get(docTokenPos).getText();
+                if (docTokText != null && docTokText.trim().length() > 0
+                        && !docTokText.equals("\n") && !docTokText.equals("\r")
+                        && !docTokText.equals(" ") && !docTokText.equals("\t")) {
+                    break;
+                }
+                docTokenPos++;
+            }
+
+            if (docTokenPos >= documentTokens.size()) {
+                break;
+            }
+
+            int currentTokenDocPos = docTokenPos;
+            docTokenPos++; // advance for next iteration
+
+            // handle label transitions
+            if (!curPlainLabel.equals(lastPlainLabel) && lastPlainLabel != null) {
+                // close the previous segment
+                if (segmentStartPos != -1 && segmentEndPos != -1 && segmentStartPos <= segmentEndPos) {
+                    int startBlockIdx = tokenToBlock[segmentStartPos];
+                    int endBlockIdx = tokenToBlock[segmentEndPos];
+                    DocumentPointer pointerA = new DocumentPointer(doc, startBlockIdx, segmentStartPos);
+                    DocumentPointer pointerB = new DocumentPointer(doc, endBlockIdx, segmentEndPos);
+                    labeledBlocks.put(lastPlainLabel, new DocumentPiece(pointerA, pointerB));
+                }
+                segmentStartPos = currentTokenDocPos;
+            }
+
+            if (segmentStartPos == -1) {
+                segmentStartPos = currentTokenDocPos;
+            }
+            segmentEndPos = currentTokenDocPos;
+
+            lastPlainLabel = curPlainLabel;
+        }
+
+        return doc;
+    }
+
     /**
      * Set the main segments of the document based on the full text parsing results
      *

grobid-core/src/main/java/org/grobid/core/document/Document.java

@@ -150,6 +150,7 @@ public void setImages(List<GraphicObject> images) {
     protected double maxBlockSpacing = 0.0;
     protected double minBlockSpacing = 0.0;
     protected int documentLenghtChar = -1; // length here is expressed as number of characters
+    protected double averageFontSize = 10.0; // average font size across all tokens
 
     // not used
     protected int beginBody = -1;
@@ -269,6 +270,10 @@ public double getMinBlockSpacing() {
         return minBlockSpacing;
     }
 
+    public double getAverageFontSize() {
+        return averageFontSize;
+    }
+
     public void setAnalyzer(Analyzer analyzer) {
         this.analyzer = analyzer;
     }
@@ -1622,11 +1627,22 @@ else if (spacing < minBlockSpacing)
             }
         }
 
-        /*System.out.println("documentLenghtChar: " + documentLenghtChar);
-        System.out.println("maxBlockSpacing: " + maxBlockSpacing);
-        System.out.println("minBlockSpacing: " + minBlockSpacing);
-        System.out.println("maxCharacterDensity: " + maxCharacterDensity);
-        System.out.println("minCharacterDensity: " + minCharacterDensity);*/
+        // compute average font size across all tokens
+        double fontSizeSum = 0;
+        int fontSizeCount = 0;
+        for (Block block : blocks) {
+            List<LayoutToken> tokens = block.getTokens();
+            if (tokens == null)
+                continue;
+            for (LayoutToken token : tokens) {
+                double fs = token.getFontSize();
+                if (fs > 0) {
+                    fontSizeSum += fs;
+                    fontSizeCount++;
+                }
+            }
+        }
+        averageFontSize = fontSizeCount > 0 ? fontSizeSum / fontSizeCount : 10.0;
     }
 
     public DocumentSource getDocumentSource() {

grobid-core/src/main/java/org/grobid/core/document/TEIFormatter.java

@@ -2406,6 +2406,24 @@ private org.grobid.core.utilities.Pair<String, String> getSectionNumber(String t
         }
     }
 
+    /**
+     * Normalize a reference label (BibDataSet.refSymbol) into a candidate @n grouping value.
+     * Strips leading/trailing punctuation and whitespace (e.g. "15." or "(15)" -> "15") and
+     * returns null when nothing usable remains.
+     *
+     * @param refSymbol the propagated reference label (footnote number), possibly null
+     * @return the normalized label, or null when blank/absent
+     */
+    protected static String deriveGroupLabel(String refSymbol) {
+        if (StringUtils.isBlank(refSymbol)) {
+            return null;
+        }
+        String label = TextUtilities
+                .removeLeadingAndTrailingChars(refSymbol, "([{<,. \n", ")}]>,.: \n")
+                .trim();
+        return StringUtils.isBlank(label) ? null : label;
+    }
+
     public StringBuilder toTEIReferences(
             StringBuilder tei,
             List<BibDataSet> bds,
@@ -2422,6 +2440,10 @@ public StringBuilder toTEIReferences(
                 for (BibDataSet bib : bds) {
                     BiblioItem bit = bib.getResBib();
                     bit.setReference(bib.getRawBib());
+                    // expose the reference label (the footnote number propagated in the
+                    // dh-law-footnotes flavour) as @n so consumers can group references that
+                    // belong to the same footnote.
+                    bit.setTeiGroupLabel(deriveGroupLabel(bib.getRefSymbol()));
                     if (bit != null) {
                         tei.append("\n" + bit.toTEI(p, 0, config));
                     } else {

grobid-core/src/main/java/org/grobid/core/engines/AbstractParser.java

@@ -33,6 +33,9 @@
 public abstract class AbstractParser implements GenericTagger, Closeable {
     private static final Logger LOGGER = LoggerFactory.getLogger(AbstractParser.class);
     private GenericTagger genericTagger;
+    // The flavor-aware model this parser was constructed with. Subclasses re-query the engine
+    // and DeLFT runtime params against this model (not against a hardcoded base constant), so
+    // that flavor overrides are honored.
     protected final GrobidModel model;
     protected GrobidAnalyzer analyzer = GrobidAnalyzer.getInstance();