Skip to content

Commit 95a6eb3

Browse files
chadgpt-o1JamesNK
chadgpt-o1
authored and
JamesNK
committed
jpath regex timeout support added for a single regex expression, global umbrella for all regex calls, and support for allowing regex calls to get compiled if necessary
require users to set a global cancellation token themselves, to avoid a potential unmanaged memory leak, which cleans up the code a bunch and fixes the failing tests fix tests so cached regex's don't cause one test to fail using prior tests timeout... handle the global timeout of multitoken regex internally and test for it
1 parent 1403f5d commit 95a6eb3

File tree

8 files changed

+199
-18
lines changed

8 files changed

+199
-18
lines changed

‎Src/Directory.Build.props

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,5 +22,7 @@
2222
<SystemValueTuplePackageVersion>4.4.0</SystemValueTuplePackageVersion>
2323
<XunitPackageVersion>2.3.1</XunitPackageVersion>
2424
<XunitRunnerVisualStudioPackageVersion>2.3.1</XunitRunnerVisualStudioPackageVersion>
25+
<BogusPackageVersion>32.0.2</BogusPackageVersion>
26+
<AsyncExPackageVersion>5.1.0</AsyncExPackageVersion>
2527
</PropertyGroup>
2628
</Project>

‎Src/Newtonsoft.Json.Tests/Linq/JsonPath/JPathExecuteTests.cs

Lines changed: 76 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,13 @@
3131
#endif
3232
using Newtonsoft.Json.Linq.JsonPath;
3333
using Newtonsoft.Json.Tests.Bson;
34+
#if HAVE_REGEX_TIMEOUTS
35+
using Bogus;
36+
using Nito.AsyncEx;
37+
using System.Text.RegularExpressions;
38+
using System.Threading.Tasks;
39+
using System.Threading;
40+
#endif
3441
#if DNXCORE50
3542
using Xunit;
3643
using Test = Xunit.FactAttribute;
@@ -44,7 +51,6 @@
4451
using Newtonsoft.Json.Utilities.LinqBridge;
4552
#else
4653
using System.Linq;
47-
4854
#endif
4955

5056
namespace Newtonsoft.Json.Tests.Linq.JsonPath
@@ -71,6 +77,75 @@ public void GreaterThanIssue1518()
7177
Assert.AreEqual(jObj, dd);
7278
}
7379

80+
#if HAVE_REGEX_TIMEOUTS
81+
[Test]
82+
public void BacktrackingRegex_SingleMatch_TimeoutRespected()
83+
{
84+
var RegexBacktrackingPattern = "(?<a>(.*?))[|].*(?<b>(.*?))[|].*(?<c>(.*?))[|].*(?<d>[1-3])[|].*(?<e>(.*?))[|].*[|].*[|].*(?<f>(.*?))[|].*[|].*(?<g>(.*?))[|].*(?<h>(.*))";
85+
86+
var faker = new Faker();
87+
var regexBacktrackingData = new JArray();
88+
89+
for (var i = 0; i < 1000; i++)
90+
{
91+
var value = $"{faker.Date.Past()}|1|{faker.Lorem.Words()}|3|{faker.Lorem.Sentences(3, ". ")}|||.\\{faker.Lorem.Word()}.cpp||{faker.Random.UShort()}|-1";
92+
regexBacktrackingData.Add(new JObject(new JProperty("b", value)));
93+
}
94+
95+
Xunit.Assert.Throws<RegexMatchTimeoutException>(() =>
96+
{
97+
var tokens = regexBacktrackingData.SelectTokens(
98+
$"[?(@.b =~ /{RegexBacktrackingPattern}/)]",
99+
errorWhenNoMatch: false,
100+
singleRegexMatchTimeout: TimeSpan.FromSeconds(.01)).ToArray();
101+
});
102+
}
103+
104+
[Test]
105+
public void BacktrackingRegex_GlobalMatch_TimeoutRespected()
106+
{
107+
var faker = new Faker();
108+
var regexBacktrackingData = new JArray();
109+
for (var i = 0; i < 1000; i++)
110+
{
111+
var value = $"{faker.Date.Past()}|1|{faker.Lorem.Words()}|3|{faker.Lorem.Sentences(3, ". ")}|||.\\{faker.Lorem.Word()}.cpp||{faker.Random.UShort()}|-1";
112+
regexBacktrackingData.Add(new JObject(new JProperty("b", value)));
113+
}
114+
var RegexBacktrackingPattern = "(?<i>(.*?))[|].*(?<b>(.*?))[|].*(?<c>(.*?))[|].*(?<d>[1-3])[|].*(?<e>(.*?))[|].*[|].*[|].*(?<f>(.*?))[|].*[|].*(?<g>(.*?))[|].*(?<h>(.*))";
115+
var jpathExpression = $"[?(@.b =~ /{RegexBacktrackingPattern}/c)]";
116+
117+
var exceptionThrow = Xunit.Assert.ThrowsAny<Exception>(() =>
118+
{
119+
var tokens = regexBacktrackingData.SelectTokens(
120+
jpathExpression,
121+
errorWhenNoMatch: false,
122+
singleRegexMatchTimeout: TimeSpan.FromSeconds(2),
123+
globalRegexMatchTimeout: TimeSpan.FromSeconds(.5)).ToArray();
124+
});
125+
Assert.IsTrue(exceptionThrow is OperationCanceledException or RegexMatchTimeoutException);
126+
}
127+
128+
[Test]
129+
public async Task BacktrackingRegexCanBeVerySlowWithoutTimeouts()
130+
{
131+
var RegexBacktrackingPattern = "(?<j>(.*?))[|].*(?<b>(.*?))[|].*(?<c>(.*?))[|].*(?<d>[1-3])[|].*(?<e>(.*?))[|].*[|].*[|].*(?<f>(.*?))[|].*[|].*(?<g>(.*?))[|].*(?<h>(.*))";
132+
var faker = new Faker();
133+
var regexBacktrackingData = new JArray();
134+
135+
for (var i = 0; i < 10; i++)
136+
{
137+
var value = $"{faker.Date.Past()}|1|{faker.Lorem.Words()}|3|{faker.Lorem.Sentences(3, ". ")}|||.\\{faker.Lorem.Word()}.cpp||{faker.Random.UShort()}|-1";
138+
regexBacktrackingData.Add(new JObject(new JProperty("b", value)));
139+
}
140+
141+
var selectTokenTask = Task.Run(() => regexBacktrackingData.SelectTokens($"[?(@.b =~ /{RegexBacktrackingPattern}/)]").ToArray());
142+
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(.3));
143+
using var ctts = new CancellationTokenTaskSource<bool>(cts.Token);
144+
var finishedTask = await Task.WhenAny(selectTokenTask, ctts.Task);
145+
Assert.AreEqual(finishedTask, ctts.Task);
146+
}
147+
#endif
148+
74149
[Test]
75150
public void GreaterThanWithIntegerParameterAndStringValue()
76151
{

‎Src/Newtonsoft.Json.Tests/Newtonsoft.Json.Tests.csproj

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
<PropertyGroup>
33
<TargetFrameworks Condition="'$(TestFrameworks)'==''">net46;net40;net35;net20;net5.0;netcoreapp3.1;netcoreapp2.1</TargetFrameworks>
44
<TargetFrameworks Condition="'$(TestFrameworks)'!=''">$(TestFrameworks)</TargetFrameworks>
5-
<LangVersion>8.0</LangVersion>
5+
<LangVersion>9.0</LangVersion>
66
<VersionPrefix>1.0</VersionPrefix>
77
<Authors>James Newton-King</Authors>
88
<Company>Newtonsoft</Company>
@@ -76,6 +76,7 @@
7676
<Reference Include="System.ComponentModel.DataAnnotations" />
7777
<Reference Include="System.Web.Extensions" />
7878
<Reference Include="System.Data.DataSetExtensions" />
79+
<PackageReference Include="Bogus" Version="$(BogusPackageVersion)" />
7980
</ItemGroup>
8081
<PropertyGroup Condition="'$(TargetFramework)'=='net40'">
8182
<AssemblyTitle>Json.NET Tests .NET 4.0</AssemblyTitle>
@@ -121,11 +122,14 @@
121122
<PackageReference Include="xunit" Version="$(XunitPackageVersion)" />
122123
<PackageReference Include="xunit.runner.visualstudio" Version="$(XunitRunnerVisualStudioPackageVersion)" />
123124
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="$(MicrosoftNETTestSdkPackageVersion)" />
125+
<PackageReference Include="Bogus" Version="$(BogusPackageVersion)" />
126+
<PackageReference Include="Nito.AsyncEx.Tasks" Version="$(AsyncExPackageVersion)" />
124127
</ItemGroup>
128+
125129
<PropertyGroup Condition="'$(TargetFramework)'=='net5.0'">
126130
<AssemblyTitle>Json.NET Tests .NET Standard 2.0</AssemblyTitle>
127131
<ReferringTargetFrameworkForProjectReferences>.NETStandard,Version=v2.0</ReferringTargetFrameworkForProjectReferences>
128-
<DefineConstants>NETSTANDARD2_0;DNXCORE50;PORTABLE;HAVE_BENCHMARKS;$(AdditionalConstants)</DefineConstants>
132+
<DefineConstants>NETSTANDARD2_0;DNXCORE50;PORTABLE;HAVE_BENCHMARKS;HAVE_REGEX_TIMEOUTS;$(AdditionalConstants)</DefineConstants>
129133
</PropertyGroup>
130134

131135
<ItemGroup Condition="'$(TargetFramework)'=='netcoreapp3.1'">
@@ -142,11 +146,13 @@
142146
<PackageReference Include="xunit" Version="$(XunitPackageVersion)" />
143147
<PackageReference Include="xunit.runner.visualstudio" Version="$(XunitRunnerVisualStudioPackageVersion)" />
144148
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="$(MicrosoftNETTestSdkPackageVersion)" />
149+
<PackageReference Include="Bogus" Version="$(BogusPackageVersion)" />
150+
<PackageReference Include="Nito.AsyncEx.Tasks" Version="$(AsyncExPackageVersion)" />
145151
</ItemGroup>
146152
<PropertyGroup Condition="'$(TargetFramework)'=='netcoreapp3.1'">
147153
<AssemblyTitle>Json.NET Tests .NET Standard 1.3</AssemblyTitle>
148154
<ReferringTargetFrameworkForProjectReferences>.NETStandard,Version=v1.3</ReferringTargetFrameworkForProjectReferences>
149-
<DefineConstants>NETSTANDARD1_3;DNXCORE50;PORTABLE;HAVE_BENCHMARKS;$(AdditionalConstants)</DefineConstants>
155+
<DefineConstants>NETSTANDARD1_3;DNXCORE50;PORTABLE;HAVE_BENCHMARKS;HAVE_REGEX_TIMEOUTS;$(AdditionalConstants)</DefineConstants>
150156
</PropertyGroup>
151157

152158
<ItemGroup Condition="'$(TargetFramework)'=='netcoreapp2.1'">
@@ -162,10 +168,13 @@
162168
<PackageReference Include="xunit" Version="$(XunitPackageVersion)" />
163169
<PackageReference Include="xunit.runner.visualstudio" Version="$(XunitRunnerVisualStudioPackageVersion)" />
164170
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="$(MicrosoftNETTestSdkPackageVersion)" />
171+
<PackageReference Include="Bogus" Version="$(BogusPackageVersion)" />
172+
<PackageReference Include="Nito.AsyncEx.Tasks" Version="$(AsyncExPackageVersion)" />
165173
</ItemGroup>
174+
166175
<PropertyGroup Condition="'$(TargetFramework)'=='netcoreapp2.1'">
167176
<AssemblyTitle>Json.NET Tests .NET Standard 1.0</AssemblyTitle>
168177
<ReferringTargetFrameworkForProjectReferences>.NETStandard,Version=v1.0</ReferringTargetFrameworkForProjectReferences>
169-
<DefineConstants>NETSTANDARD1_0;DNXCORE50;PORTABLE;$(AdditionalConstants)</DefineConstants>
178+
<DefineConstants>NETSTANDARD1_0;DNXCORE50;PORTABLE;HAVE_REGEX_TIMEOUTS;$(AdditionalConstants)</DefineConstants>
170179
</PropertyGroup>
171180
</Project>

‎Src/Newtonsoft.Json/Linq/JToken.cs

Lines changed: 76 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@
4444
using Newtonsoft.Json.Utilities.LinqBridge;
4545
#else
4646
using System.Linq;
47+
using System.Threading;
4748
#endif
4849

4950
namespace Newtonsoft.Json.Linq
@@ -485,7 +486,7 @@ private static bool ValidateToken(JToken o, JTokenType[] validTypes, bool nullab
485486
return (Array.IndexOf(validTypes, o.Type) != -1) || (nullable && (o.Type == JTokenType.Null || o.Type == JTokenType.Undefined));
486487
}
487488

488-
#region Cast from operators
489+
#region Cast from operators
489490
/// <summary>
490491
/// Performs an explicit conversion from <see cref="Newtonsoft.Json.Linq.JToken"/> to <see cref="System.Boolean"/>.
491492
/// </summary>
@@ -1497,9 +1498,9 @@ private static BigInteger ToBigInteger(JToken value)
14971498
return ConvertUtils.ToBigInteger(v.Value);
14981499
}
14991500
#endif
1500-
#endregion
1501+
#endregion
15011502

1502-
#region Cast to operators
1503+
#region Cast to operators
15031504
/// <summary>
15041505
/// Performs an implicit conversion from <see cref="Boolean"/> to <see cref="JToken"/>.
15051506
/// </summary>
@@ -1863,7 +1864,7 @@ public static implicit operator JToken(Guid? value)
18631864
{
18641865
return new JValue(value);
18651866
}
1866-
#endregion
1867+
#endregion
18671868

18681869
IEnumerator IEnumerable.GetEnumerator()
18691870
{
@@ -2336,6 +2337,35 @@ int IJsonLineInfo.LinePosition
23362337
return token;
23372338
}
23382339

2340+
#if HAVE_REGEX_TIMEOUTS
2341+
/// <summary>
2342+
/// Selects a <see cref="JToken"/> using a JSONPath expression. Selects the token that matches the object path.
2343+
/// </summary>
2344+
/// <param name="path">
2345+
/// A <see cref="String"/> that contains a JSONPath expression.
2346+
/// </param>
2347+
/// <param name="errorWhenNoMatch">A flag to indicate whether an error should be thrown if no tokens are found when evaluating part of the expression.</param>
2348+
/// <param name="singleMatchTimeout">the time after which a single call to regex.ismatch must complete, default is forever</param>
2349+
/// <returns>A <see cref="JToken"/>.</returns>
2350+
public JToken? SelectToken(string path, bool errorWhenNoMatch, TimeSpan? singleMatchTimeout = default)
2351+
{
2352+
JPath p = new JPath(path, singleMatchTimeout);
2353+
2354+
JToken? token = null;
2355+
foreach (JToken t in p.Evaluate(this, this, errorWhenNoMatch))
2356+
{
2357+
if (token != null)
2358+
{
2359+
throw new JsonException("Path returned multiple tokens.");
2360+
}
2361+
2362+
token = t;
2363+
}
2364+
2365+
return token;
2366+
}
2367+
#endif
2368+
23392369
/// <summary>
23402370
/// Selects a collection of elements using a JSONPath expression.
23412371
/// </summary>
@@ -2358,10 +2388,51 @@ public IEnumerable<JToken> SelectTokens(string path)
23582388
/// <returns>An <see cref="IEnumerable{T}"/> of <see cref="JToken"/> that contains the selected elements.</returns>
23592389
public IEnumerable<JToken> SelectTokens(string path, bool errorWhenNoMatch)
23602390
{
2361-
JPath p = new JPath(path);
2391+
var p = new JPath(path);
23622392
return p.Evaluate(this, this, errorWhenNoMatch);
23632393
}
23642394

2395+
#if HAVE_REGEX_TIMEOUTS
2396+
/// <summary>
2397+
/// Selects a collection of elements using a JSONPath expression.
2398+
/// </summary>
2399+
/// <param name="path">
2400+
/// A <see cref="String"/> that contains a JSONPath expression.
2401+
/// </param>
2402+
/// <param name="errorWhenNoMatch">A flag to indicate whether an error should be thrown if no tokens are found when evaluating part of the expression.</param>
2403+
/// <param name="singleRegexMatchTimeout">
2404+
/// for every token that matches a jpath, the time a regex is permitted to run
2405+
/// against that token before timeout
2406+
/// </param>
2407+
/// <param name="globalRegexMatchTimeout">
2408+
/// the time this method should wait for the given jpath regex to match all tokens.
2409+
/// worst case expected execution time roughly globalRegexMatchTimeout + Min(singleRegexMatchTimeout, globalRegexMatchTimeout)
2410+
/// </param>
2411+
/// <exception cref="System.Text.RegularExpressions.RegexMatchTimeoutException">if single call timeout exceeded</exception>"
2412+
/// <returns>An <see cref="IEnumerable{T}"/> of <see cref="JToken"/> that contains the selected elements.</returns>
2413+
public IEnumerable<JToken> SelectTokens(string path,
2414+
bool errorWhenNoMatch,
2415+
TimeSpan? singleRegexMatchTimeout = default,
2416+
TimeSpan? globalRegexMatchTimeout = default)
2417+
{
2418+
var singleTimeout = singleRegexMatchTimeout ?? Timeout.InfiniteTimeSpan;
2419+
var globalTimeout = globalRegexMatchTimeout ?? Timeout.InfiniteTimeSpan;
2420+
if (globalTimeout != Timeout.InfiniteTimeSpan && globalTimeout < singleTimeout)
2421+
{
2422+
singleTimeout = globalTimeout;
2423+
}
2424+
2425+
using var cts = new CancellationTokenSource(globalTimeout);
2426+
var p = new JPath(path, singleTimeout);
2427+
var results = p.Evaluate(this, this, errorWhenNoMatch);
2428+
foreach (var result in results)
2429+
{
2430+
cts.Token.ThrowIfCancellationRequested();
2431+
yield return result;
2432+
}
2433+
}
2434+
#endif
2435+
23652436
#if HAVE_DYNAMIC
23662437
/// <summary>
23672438
/// Returns the <see cref="DynamicMetaObject"/> responsible for binding operations performed on this object.

‎Src/Newtonsoft.Json/Linq/JsonPath/JPath.cs

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,8 @@
2727
using System.Collections.Generic;
2828
using System.Globalization;
2929
using System.Text;
30+
using System.Text.RegularExpressions;
31+
using System.Threading;
3032
using Newtonsoft.Json.Utilities;
3133

3234
namespace Newtonsoft.Json.Linq.JsonPath
@@ -37,14 +39,16 @@ internal class JPath
3739

3840
private readonly string _expression;
3941
public List<PathFilter> Filters { get; }
42+
public readonly TimeSpan? RegexSingleMatchTimeout = default;
4043

4144
private int _currentIndex;
4245

43-
public JPath(string expression)
46+
public JPath(string expression, TimeSpan? regexSingleMatchTimeout = default)
4447
{
4548
ValidationUtils.ArgumentNotNull(expression, nameof(expression));
4649
_expression = expression;
4750
Filters = new List<PathFilter>();
51+
RegexSingleMatchTimeout = regexSingleMatchTimeout;
4852

4953
ParseMain();
5054
}
@@ -509,7 +513,8 @@ private QueryExpression ParseExpression()
509513
right = ParseSide();
510514
}
511515

512-
BooleanQueryExpression booleanExpression = new BooleanQueryExpression(op, left, right);
516+
517+
var booleanExpression = new BooleanQueryExpression(op, left, right, RegexSingleMatchTimeout);
513518

514519
if (_expression[_currentIndex] == ')')
515520
{

‎Src/Newtonsoft.Json/Linq/JsonPath/QueryExpression.cs

Lines changed: 17 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
using System.IO;
55
using System.Text.RegularExpressions;
66
using System.Diagnostics;
7+
using System.Threading;
78
#if !HAVE_LINQ
89
using Newtonsoft.Json.Utilities.LinqBridge;
910
#else
@@ -83,11 +84,17 @@ internal class BooleanQueryExpression : QueryExpression
8384
{
8485
public readonly object Left;
8586
public readonly object? Right;
87+
public readonly TimeSpan? SingleMatchTimeout;
8688

87-
public BooleanQueryExpression(QueryOperator @operator, object left, object? right) : base(@operator)
89+
public BooleanQueryExpression(QueryOperator @operator,
90+
object left,
91+
object? right,
92+
TimeSpan? regexSingleMatchTimeout = default)
93+
: base(@operator)
8894
{
8995
Left = left;
9096
Right = right;
97+
SingleMatchTimeout = regexSingleMatchTimeout;
9198
}
9299

93100
private IEnumerable<JToken> GetResult(JToken root, JToken t, object? o)
@@ -143,7 +150,7 @@ private bool MatchTokens(JToken leftResult, JToken rightResult)
143150
switch (Operator)
144151
{
145152
case QueryOperator.RegexEquals:
146-
if (RegexEquals(leftValue, rightValue))
153+
if (RegexEquals(leftValue, rightValue, SingleMatchTimeout))
147154
{
148155
return true;
149156
}
@@ -215,7 +222,9 @@ private bool MatchTokens(JToken leftResult, JToken rightResult)
215222
return false;
216223
}
217224

218-
private static bool RegexEquals(JValue input, JValue pattern)
225+
private static bool RegexEquals(JValue input,
226+
JValue pattern,
227+
TimeSpan? regexMatchTimeout = default)
219228
{
220229
if (input.Type != JTokenType.String || pattern.Type != JTokenType.String)
221230
{
@@ -228,7 +237,12 @@ private static bool RegexEquals(JValue input, JValue pattern)
228237
string patternText = regexText.Substring(1, patternOptionDelimiterIndex - 1);
229238
string optionsText = regexText.Substring(patternOptionDelimiterIndex + 1);
230239

240+
#if HAVE_REGEX_TIMEOUTS
241+
var timeout = regexMatchTimeout ?? Regex.InfiniteMatchTimeout;
242+
return Regex.IsMatch((string)input.Value!, patternText, MiscellaneousUtils.GetRegexOptions(optionsText), timeout);
243+
#else
231244
return Regex.IsMatch((string)input.Value!, patternText, MiscellaneousUtils.GetRegexOptions(optionsText));
245+
#endif
232246
}
233247

234248
internal static bool EqualsWithStringCoercion(JValue value, JValue queryValue)

0 commit comments

Comments
 (0)