diff --git a/build.gradle b/build.gradle index 5d29fb3bf9..b50596e616 100644 --- a/build.gradle +++ b/build.gradle @@ -260,6 +260,11 @@ tasks.withType(JavaCompile).configureEach { // Test execution configuration with native access and adequate heap tasks.withType(Test).configureEach { jvmArgs += '--enable-native-access=ALL-UNNAMED' + // PerlScriptExecutionTest runs Perl directly in the Gradle worker rather + // than through jperl.bat. Match the launcher's recursion-safe stack size, + // especially on Windows where the default worker stack is too small for + // the 1000-call compatibility guard. + jvmArgs += '-Xss16m' // Netty still uses the transitional sun.misc.Unsafe memory API. PerlOnJava // requires Java 24, where explicitly allowing it suppresses the terminal // deprecation banner while retaining the tested Netty allocation path. diff --git a/docs/about/changelog.md b/docs/about/changelog.md index 6406c88af2..c433443d01 100644 --- a/docs/about/changelog.md +++ b/docs/about/changelog.md @@ -6,6 +6,8 @@ priorities and future plans. ## Work in progress +- Fixed large dynamic named-subexpression grammars hanging during regex compilation. + - Preserve Data::Dumper's pure-Perl numeric-string behavior for Test::Differences, including copied `qw` values and numeric zero fixtures. diff --git a/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java b/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java index afbc24abf1..c3e040c170 100644 --- a/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java +++ b/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java @@ -413,6 +413,7 @@ public boolean supportsPositions() { namedCharacterCache, namedCharacterSourceMode, userPropertyState); int options = toJoniOptions(flags, forceAsciiClasses, perlReStrict); + if (trustedCalloutCount > 0) options |= Option.PERL_DYNAMIC_CALLOUT_SOURCE; if (byteMode && byteBackedPattern) options |= Option.PERL_BYTE_PATTERN; userPropertyPackage = UnicodeResolver.activeUserPropertyPackage(); regex = new Regex(bytes, 0, bytes.length, options, diff --git a/src/test/resources/unit/regex_large_named_grammar.t b/src/test/resources/unit/regex_large_named_grammar.t new file mode 100644 index 0000000000..334b4e1910 --- /dev/null +++ b/src/test/resources/unit/regex_large_named_grammar.t @@ -0,0 +1,20 @@ +use strict; +use warnings; +use re 'eval'; + +our $dynamic_leaf = 'x'; + +my @definitions; +for my $number (0 .. 17) { + my $next = $number + 1; + push @definitions, "(?(?&rule$next)(?&rule$next)?)"; +} +push @definitions, '(?(??{ $dynamic_leaf }))'; + +my $grammar = '\\A(?&rule0)\\z(?(DEFINE)' . join('', @definitions) . ')'; +my $compiled = qr/$grammar/; + +print "1..2\n"; +print "ok 1 - compiles a large named-subexpression grammar\n"; +print "not " unless 'x' =~ $compiled; +print "ok 2 - compiled grammar matches its leaf\n"; diff --git a/third_party/joni/src/org/joni/Analyser.java b/third_party/joni/src/org/joni/Analyser.java index 6001994eaa..3a3f8308b4 100644 --- a/third_party/joni/src/org/joni/Analyser.java +++ b/third_party/joni/src/org/joni/Analyser.java @@ -34,8 +34,13 @@ import java.util.IllegalFormatConversionException; import java.util.ArrayList; +import java.util.ArrayDeque; import java.util.Collections; +import java.util.Deque; +import java.util.HashSet; import java.util.IdentityHashMap; +import java.util.List; +import java.util.Map; import java.util.Set; import org.jcodings.CaseFoldCodeItem; @@ -64,6 +69,8 @@ import org.joni.exception.ValueException; final class Analyser extends Parser { + private Map recursiveHeadResults; + private Map recursiveNonHeadResults; protected Analyser(Regex regex, Syntax syntax, byte[]bytes, int p, int end, WarnCallback warnings) { this(regex, syntax, bytes, p, end, warnings, true); @@ -110,10 +117,15 @@ protected final void compile() { if (env.numCall > 0) { env.unsetAddrList = new UnsetAddrList(env.numCall); setupSubExpCall(root); - // r != 0 ??? - subexpRecursiveCheckTrav(root); - // r < 0 -< err, FOUND_CALLED_NODE = 1 - subexpInfRecursiveCheckTrav(root); + markSubexpressionRecursion(root); + markSubexpressionReferences(root); + // Dynamic pattern callouts have runtime-defined width, so a + // static zero-width recursion proof is not sound for them. + if (!Option.isPerlDynamicCalloutSource(env.option) + && !containsDynamicCallout(root) && !env.parsedProgramMetadata().has( + Regex.ParsedProgramFeature.DYNAMIC_CALLOUT)) { + subexpInfRecursiveCheckTrav(root); + } // r != 0 recursion infinite ??? regex.numCall = env.numCall; } else { @@ -1176,6 +1188,16 @@ private boolean checkTypeTree(Node node, int typeMask, int encloseMask, int anch private static final int RECURSION_EXIST = 1; private static final int RECURSION_INFINITE = 2; private int subexpInfRecursiveCheck(Node node, boolean head) { + Map results = head ? recursiveHeadResults : recursiveNonHeadResults; + if (results != null && results.containsKey(node)) return results.get(node); + int result = subexpInfRecursiveCheckUncached(node, head); + // A nonzero result depends on the active recursion path. Only a + // proven non-recursive subtree is valid to share between paths. + if (results != null && result == 0) results.put(node, result); + return result; + } + + private int subexpInfRecursiveCheckUncached(Node node, boolean head) { int r = 0; switch (node.getType()) { @@ -1188,7 +1210,9 @@ private int subexpInfRecursiveCheck(Node node, boolean head) { r |= ret; if (head) { min = getMinMatchLength(x.value); - if (min != 0) head = false; + // A dynamic pattern callout has runtime-defined width. + // It cannot prove that a recursive path is zero-width. + if (min != 0 || containsDynamicCallout(x.value)) head = false; } } while ((x = x.tail) != null); break; @@ -1283,9 +1307,13 @@ protected final int subexpInfRecursiveCheckTrav(Node node) { EncloseNode en = (EncloseNode)node; if (en.isRecursion()) { en.setMark1(); + recursiveHeadResults = new IdentityHashMap<>(); + recursiveNonHeadResults = new IdentityHashMap<>(); r = subexpInfRecursiveCheck(en.target, true); if (r > 0) newValueException(NEVER_ENDING_RECURSION); en.clearMark1(); + recursiveHeadResults = null; + recursiveNonHeadResults = null; } r = subexpInfRecursiveCheckTrav(en.target); break; @@ -1393,15 +1421,7 @@ protected final int subexpRecursiveCheckTrav(Node node) { case NodeType.ENCLOSE: EncloseNode en = (EncloseNode)node; - if (!en.isRecursion()) { - if (en.isCalled()) { - en.setMark1(); - r = subexpRecursiveCheck(en.target); - if (r != 0) en.setRecursion(); - en.clearMark1(); - } - } - r = subexpRecursiveCheckTrav(en.target); + r = markSubexpressionReferences(en.target); if (en.isCalled()) r |= FOUND_CALLED_NODE; break; @@ -1412,11 +1432,145 @@ protected final int subexpRecursiveCheckTrav(Node node) { return r; } + /** Retains quantifier reference metadata without restarting call-graph searches. */ + private int markSubexpressionReferences(Node node) { + return subexpRecursiveCheckTrav(node); + } + private static boolean isImpossibleQuantifier(QuantifierNode quantifier) { return !isRepeatInfinite(quantifier.upper) && quantifier.lower > quantifier.upper; } + /** + * Mark recursive subexpression calls by finding strongly connected + * components in the call graph. The old tree walk restarted a path search + * from every called group. A grammar with many shared named definitions + * therefore revisited the same call paths exponentially often. + */ + private void markSubexpressionRecursion(Node root) { + Map> calls = new IdentityHashMap<>(); + collectSubexpressionCalls(root, new ArrayList<>(), calls); + + Map index = new IdentityHashMap<>(); + Map lowlink = new IdentityHashMap<>(); + Map component = new IdentityHashMap<>(); + Set onStack = Collections.newSetFromMap(new IdentityHashMap<>()); + Deque stack = new ArrayDeque<>(); + int[] nextIndex = {0}; + int[] nextComponent = {0}; + Set recursiveComponents = new HashSet<>(); + + for (EncloseNode group : calls.keySet()) { + if (!index.containsKey(group)) { + findSubexpressionComponents(group, calls, index, lowlink, component, + onStack, stack, nextIndex, nextComponent, recursiveComponents); + } + } + + for (Map.Entry> entry : calls.entrySet()) { + Integer sourceComponent = component.get(entry.getKey()); + for (CallNode call : entry.getValue()) { + Integer targetComponent = component.get(call.target); + if (sourceComponent != null && sourceComponent.equals(targetComponent) + && recursiveComponents.contains(sourceComponent)) { + call.setRecursion(); + } + } + } + } + + private void collectSubexpressionCalls(Node node, List enclosingGroups, + Map> calls) { + switch (node.getType()) { + case NodeType.LIST: + case NodeType.ALT: + for (ListNode list = (ListNode) node; list != null; list = list.tail) { + collectSubexpressionCalls(list.value, enclosingGroups, calls); + } + break; + case NodeType.QTFR: + collectSubexpressionCalls(((QuantifierNode) node).target, enclosingGroups, calls); + break; + case NodeType.ANCHOR: + AnchorNode anchor = (AnchorNode) node; + if (anchor.target != null) collectSubexpressionCalls(anchor.target, enclosingGroups, calls); + break; + case NodeType.ENCLOSE: + EncloseNode enclosure = (EncloseNode) node; + if (enclosure.assertionCondition != null) { + collectSubexpressionCalls(enclosure.assertionCondition, enclosingGroups, calls); + } + if (enclosure.isMemory()) { + calls.computeIfAbsent(enclosure, ignored -> new ArrayList<>()); + enclosingGroups.add(enclosure); + collectSubexpressionCalls(enclosure.target, enclosingGroups, calls); + enclosingGroups.remove(enclosingGroups.size() - 1); + } else { + collectSubexpressionCalls(enclosure.target, enclosingGroups, calls); + } + break; + case NodeType.CALL: + CallNode call = (CallNode) node; + for (EncloseNode group : enclosingGroups) { + calls.computeIfAbsent(group, ignored -> new ArrayList<>()).add(call); + } + break; + default: + break; + } + } + + private void findSubexpressionComponents(EncloseNode group, + Map> calls, Map index, + Map lowlink, Map component, + Set onStack, Deque stack, int[] nextIndex, + int[] nextComponent, Set recursiveComponents) { + int groupIndex = nextIndex[0]++; + index.put(group, groupIndex); + lowlink.put(group, groupIndex); + stack.push(group); + onStack.add(group); + + for (CallNode call : calls.getOrDefault(group, List.of())) { + EncloseNode target = call.target; + if (!index.containsKey(target)) { + findSubexpressionComponents(target, calls, index, lowlink, component, + onStack, stack, nextIndex, nextComponent, recursiveComponents); + lowlink.put(group, Math.min(lowlink.get(group), lowlink.get(target))); + } else if (onStack.contains(target)) { + lowlink.put(group, Math.min(lowlink.get(group), index.get(target))); + } + } + + if (!lowlink.get(group).equals(index.get(group))) return; + + int componentId = nextComponent[0]++; + int size = 0; + boolean selfCall = false; + EncloseNode member; + do { + member = stack.pop(); + onStack.remove(member); + component.put(member, componentId); + size++; + } while (member != group); + if (size == 1) { + for (CallNode call : calls.getOrDefault(group, List.of())) { + if (call.target == group) { + selfCall = true; + break; + } + } + } + if (size > 1 || selfCall) { + recursiveComponents.add(componentId); + for (Map.Entry entry : component.entrySet()) { + if (entry.getValue() == componentId) entry.getKey().setRecursion(); + } + } + } + private void setCallAttr(CallNode cn) { EncloseNode en = cn.lexicalTarget != null ? cn.lexicalTarget diff --git a/third_party/joni/src/org/joni/Option.java b/third_party/joni/src/org/joni/Option.java index 11e9c53edb..82912224c4 100644 --- a/third_party/joni/src/org/joni/Option.java +++ b/third_party/joni/src/org/joni/Option.java @@ -64,8 +64,10 @@ public final class Option { public static final int PERL_LOCALE_NON_UTF8 = (1 << 26); /** Perl 5.44 experimental enhanced /xx character-class parsing. */ public static final int PERL_ENHANCED_XX = (1 << 27); + /** The host pattern includes runtime-defined dynamic callout slots. */ + public static final int PERL_DYNAMIC_CALLOUT_SOURCE = (1 << 28); - public static final int MAXBIT = (1 << 28); /* limit */ + public static final int MAXBIT = (1 << 29); /* limit */ public static final int DEFAULT = NONE; @@ -93,6 +95,7 @@ public static String toString(int option) { if (isPerlUnicodeCharset(option)) options += "PERL_UNICODE_CHARSET"; if (isPerlLocaleNonUtf8(option)) options += "PERL_LOCALE_NON_UTF8"; if (isPerlEnhancedXx(option)) options += "PERL_ENHANCED_XX"; + if (isPerlDynamicCalloutSource(option)) options += "PERL_DYNAMIC_CALLOUT_SOURCE"; return options; } @@ -112,6 +115,10 @@ public static boolean isPerlEnhancedXx(int option) { return (option & PERL_ENHANCED_XX) != 0; } + public static boolean isPerlDynamicCalloutSource(int option) { + return (option & PERL_DYNAMIC_CALLOUT_SOURCE) != 0; + } + public static boolean isPerlReStrict(int option) { return (option & PERL_RE_STRICT) != 0; }