Merge pull request #90 from kares/expansion-limit-checks

prevent regexp expansion size overflow

Author: headius

src/org/joni/ArrayCompiler.java

@@ -443,6 +443,12 @@ private void compileRangeRepeatNode(QuantifierNode qn, int targetLen, int emptyI
 
     private static final int QUANTIFIER_EXPAND_LIMIT_SIZE   = 50; // was 50
 
+    // MRI: regcomp.c IS_EXPAND_LIMIT_OK (since Ruby 4.0.8; 3.4.12 and 3.3.13)
+    // prevent nested repeats from bypassing the limit and exhausting heap
+    private static boolean isExpandLimitOK(int tlen, int n) {
+        return n <= 0 || tlen <= QUANTIFIER_EXPAND_LIMIT_SIZE / n;
+    }
+
     private static boolean cknOn(int ckn) {
         return ckn > 0;
     }
@@ -660,7 +666,7 @@ private int compileNonCECLengthQuantifierNode(QuantifierNode qn) {
         }
 
         int len;
-        if (infinite && (qn.lower <= 1 || tlen * qn.lower <= QUANTIFIER_EXPAND_LIMIT_SIZE)) {
+        if (infinite && (qn.lower <= 1 || isExpandLimitOK(tlen, qn.lower))) {
             if (qn.lower == 1 && tlen > QUANTIFIER_EXPAND_LIMIT_SIZE) {
                 len = OPSize.JUMP;
             } else {
@@ -682,7 +688,7 @@ private int compileNonCECLengthQuantifierNode(QuantifierNode qn) {
         } else if (qn.upper == 0 && qn.isRefered) { /* /(?<n>..){0}/ */
             len = OPSize.JUMP + tlen;
         } else if (!infinite && qn.greedy &&
-                  (qn.upper == 1 || (tlen + OPSize.PUSH) * qn.upper <= QUANTIFIER_EXPAND_LIMIT_SIZE )) {
+                  (qn.upper == 1 || isExpandLimitOK(tlen + OPSize.PUSH, qn.upper))) {
             len = tlen * qn.lower;
             len += (OPSize.PUSH + tlen) * (qn.upper - qn.lower);
         } else if (!qn.greedy && qn.upper == 1 && qn.lower == 0) { /* '??' */
@@ -728,7 +734,7 @@ protected void compileNonCECQuantifierNode(QuantifierNode qn) {
         } else {
             modTLen = tlen;
         }
-        if (infinite && (qn.lower <= 1 || tlen * qn.lower <= QUANTIFIER_EXPAND_LIMIT_SIZE)) {
+        if (infinite && (qn.lower <= 1 || isExpandLimitOK(tlen, qn.lower))) {
             if (qn.lower == 1 && tlen > QUANTIFIER_EXPAND_LIMIT_SIZE) {
                 if (qn.greedy) {
                     if (qn.headExact != null) {
@@ -772,7 +778,7 @@ protected void compileNonCECQuantifierNode(QuantifierNode qn) {
             addOpcodeRelAddr(OPCode.JUMP, tlen);
             compileTree(qn.target);
         } else if (!infinite && qn.greedy &&
-                  (qn.upper == 1 || (tlen + OPSize.PUSH) * qn.upper <= QUANTIFIER_EXPAND_LIMIT_SIZE)) {
+                  (qn.upper == 1 || isExpandLimitOK(tlen + OPSize.PUSH, qn.upper))) {
             int n = qn.upper - qn.lower;
             compileTreeNTimes(qn.target, qn.lower);
 

test/org/joni/test/TestCornerCases.java

@@ -19,9 +19,14 @@
  */
 package org.joni.test;
 
+import static org.junit.Assert.assertEquals;
+
+import java.nio.charset.StandardCharsets;
+
 import org.jcodings.Encoding;
 import org.jcodings.specific.ASCIIEncoding;
 import org.joni.Config;
+import org.joni.Matcher;
 import org.joni.Option;
 import org.joni.Regex;
 import org.joni.Syntax;
@@ -56,4 +61,16 @@ public void test() throws Exception {
             nfail++;
         }
     }
+
+    @org.junit.Test(timeout = 30000)
+    public void testNestedRepeatExpansionOverflow() {
+        byte[] input = "xxxxxxxxxx".getBytes(StandardCharsets.US_ASCII);
+        for (String source : new String[]{"(?:.{90000,}){90000}", "(?:.{46341,}){46341}",
+                                          "(?:.{90000,}){90000,}", "(?:.{46341,}){46341,}"}) {
+            byte[] pattern = source.getBytes(StandardCharsets.US_ASCII);
+            Regex regex = new Regex(pattern, 0, pattern.length, Option.DEFAULT, ASCIIEncoding.INSTANCE, Syntax.DEFAULT);
+            assertEquals(source, Matcher.FAILED, regex.matcher(input).search(0, input.length, Option.NONE));
+        }
+    }
+
 }