Skip to content

Commit 42139ea

Browse files
JamesNKJamesNK
committed
Add JsonSelectSettings and regex timeout
1 parent 95a6eb3 commit 42139ea

20 files changed

+108
-193
lines changed

‎Src/Directory.Build.props

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,5 @@
2222
<SystemValueTuplePackageVersion>4.4.0</SystemValueTuplePackageVersion>
2323
<XunitPackageVersion>2.3.1</XunitPackageVersion>
2424
<XunitRunnerVisualStudioPackageVersion>2.3.1</XunitRunnerVisualStudioPackageVersion>
25-
<BogusPackageVersion>32.0.2</BogusPackageVersion>
26-
<AsyncExPackageVersion>5.1.0</AsyncExPackageVersion>
2725
</PropertyGroup>
2826
</Project>

‎Src/Newtonsoft.Json.Tests/Linq/JsonPath/JPathExecuteTests.cs

Lines changed: 8 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -32,11 +32,7 @@
3232
using Newtonsoft.Json.Linq.JsonPath;
3333
using Newtonsoft.Json.Tests.Bson;
3434
#if HAVE_REGEX_TIMEOUTS
35-
using Bogus;
36-
using Nito.AsyncEx;
3735
using System.Text.RegularExpressions;
38-
using System.Threading.Tasks;
39-
using System.Threading;
4036
#endif
4137
#if DNXCORE50
4238
using Xunit;
@@ -81,68 +77,20 @@ public void GreaterThanIssue1518()
8177
[Test]
8278
public void BacktrackingRegex_SingleMatch_TimeoutRespected()
8379
{
84-
var RegexBacktrackingPattern = "(?<a>(.*?))[|].*(?<b>(.*?))[|].*(?<c>(.*?))[|].*(?<d>[1-3])[|].*(?<e>(.*?))[|].*[|].*[|].*(?<f>(.*?))[|].*[|].*(?<g>(.*?))[|].*(?<h>(.*))";
80+
const string RegexBacktrackingPattern = "(?<a>(.*?))[|].*(?<b>(.*?))[|].*(?<c>(.*?))[|].*(?<d>[1-3])[|].*(?<e>(.*?))[|].*[|].*[|].*(?<f>(.*?))[|].*[|].*(?<g>(.*?))[|].*(?<h>(.*))";
8581

86-
var faker = new Faker();
8782
var regexBacktrackingData = new JArray();
83+
regexBacktrackingData.Add(new JObject(new JProperty("b", @"15/04/2020 8:18:03 PM|1|System.String[]|3|Libero eligendi magnam ut inventore.. Quaerat et sit voluptatibus repellendus blanditiis aliquam ut.. Quidem qui ut sint in ex et tempore.|||.\iste.cpp||46018|-1")));
8884

89-
for (var i = 0; i < 1000; i++)
85+
ExceptionAssert.Throws<RegexMatchTimeoutException>(() =>
9086
{
91-
var value = $"{faker.Date.Past()}|1|{faker.Lorem.Words()}|3|{faker.Lorem.Sentences(3, ". ")}|||.\\{faker.Lorem.Word()}.cpp||{faker.Random.UShort()}|-1";
92-
regexBacktrackingData.Add(new JObject(new JProperty("b", value)));
93-
}
94-
95-
Xunit.Assert.Throws<RegexMatchTimeoutException>(() =>
96-
{
97-
var tokens = regexBacktrackingData.SelectTokens(
87+
regexBacktrackingData.SelectTokens(
9888
$"[?(@.b =~ /{RegexBacktrackingPattern}/)]",
99-
errorWhenNoMatch: false,
100-
singleRegexMatchTimeout: TimeSpan.FromSeconds(.01)).ToArray();
101-
});
102-
}
103-
104-
[Test]
105-
public void BacktrackingRegex_GlobalMatch_TimeoutRespected()
106-
{
107-
var faker = new Faker();
108-
var regexBacktrackingData = new JArray();
109-
for (var i = 0; i < 1000; i++)
110-
{
111-
var value = $"{faker.Date.Past()}|1|{faker.Lorem.Words()}|3|{faker.Lorem.Sentences(3, ". ")}|||.\\{faker.Lorem.Word()}.cpp||{faker.Random.UShort()}|-1";
112-
regexBacktrackingData.Add(new JObject(new JProperty("b", value)));
113-
}
114-
var RegexBacktrackingPattern = "(?<i>(.*?))[|].*(?<b>(.*?))[|].*(?<c>(.*?))[|].*(?<d>[1-3])[|].*(?<e>(.*?))[|].*[|].*[|].*(?<f>(.*?))[|].*[|].*(?<g>(.*?))[|].*(?<h>(.*))";
115-
var jpathExpression = $"[?(@.b =~ /{RegexBacktrackingPattern}/c)]";
116-
117-
var exceptionThrow = Xunit.Assert.ThrowsAny<Exception>(() =>
118-
{
119-
var tokens = regexBacktrackingData.SelectTokens(
120-
jpathExpression,
121-
errorWhenNoMatch: false,
122-
singleRegexMatchTimeout: TimeSpan.FromSeconds(2),
123-
globalRegexMatchTimeout: TimeSpan.FromSeconds(.5)).ToArray();
89+
new JsonSelectSettings
90+
{
91+
RegexMatchTimeout = TimeSpan.FromSeconds(0.01)
92+
}).ToArray();
12493
});
125-
Assert.IsTrue(exceptionThrow is OperationCanceledException or RegexMatchTimeoutException);
126-
}
127-
128-
[Test]
129-
public async Task BacktrackingRegexCanBeVerySlowWithoutTimeouts()
130-
{
131-
var RegexBacktrackingPattern = "(?<j>(.*?))[|].*(?<b>(.*?))[|].*(?<c>(.*?))[|].*(?<d>[1-3])[|].*(?<e>(.*?))[|].*[|].*[|].*(?<f>(.*?))[|].*[|].*(?<g>(.*?))[|].*(?<h>(.*))";
132-
var faker = new Faker();
133-
var regexBacktrackingData = new JArray();
134-
135-
for (var i = 0; i < 10; i++)
136-
{
137-
var value = $"{faker.Date.Past()}|1|{faker.Lorem.Words()}|3|{faker.Lorem.Sentences(3, ". ")}|||.\\{faker.Lorem.Word()}.cpp||{faker.Random.UShort()}|-1";
138-
regexBacktrackingData.Add(new JObject(new JProperty("b", value)));
139-
}
140-
141-
var selectTokenTask = Task.Run(() => regexBacktrackingData.SelectTokens($"[?(@.b =~ /{RegexBacktrackingPattern}/)]").ToArray());
142-
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(.3));
143-
using var ctts = new CancellationTokenTaskSource<bool>(cts.Token);
144-
var finishedTask = await Task.WhenAny(selectTokenTask, ctts.Task);
145-
Assert.AreEqual(finishedTask, ctts.Task);
14694
}
14795
#endif
14896

‎Src/Newtonsoft.Json.Tests/Newtonsoft.Json.Tests.csproj

Lines changed: 1 addition & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@
6262
<PropertyGroup Condition="'$(TargetFramework)'=='net46'">
6363
<AssemblyTitle>Json.NET Tests</AssemblyTitle>
6464
<ReferringTargetFrameworkForProjectReferences>.NETFramework,Version=v4.5</ReferringTargetFrameworkForProjectReferences>
65-
<DefineConstants>NET45;HAVE_BENCHMARKS;$(AdditionalConstants)</DefineConstants>
65+
<DefineConstants>NET45;HAVE_BENCHMARKS;HAVE_REGEX_TIMEOUTS;$(AdditionalConstants)</DefineConstants>
6666
</PropertyGroup>
6767

6868
<ItemGroup Condition="'$(TargetFramework)'=='net40'">
@@ -76,7 +76,6 @@
7676
<Reference Include="System.ComponentModel.DataAnnotations" />
7777
<Reference Include="System.Web.Extensions" />
7878
<Reference Include="System.Data.DataSetExtensions" />
79-
<PackageReference Include="Bogus" Version="$(BogusPackageVersion)" />
8079
</ItemGroup>
8180
<PropertyGroup Condition="'$(TargetFramework)'=='net40'">
8281
<AssemblyTitle>Json.NET Tests .NET 4.0</AssemblyTitle>
@@ -122,10 +121,7 @@
122121
<PackageReference Include="xunit" Version="$(XunitPackageVersion)" />
123122
<PackageReference Include="xunit.runner.visualstudio" Version="$(XunitRunnerVisualStudioPackageVersion)" />
124123
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="$(MicrosoftNETTestSdkPackageVersion)" />
125-
<PackageReference Include="Bogus" Version="$(BogusPackageVersion)" />
126-
<PackageReference Include="Nito.AsyncEx.Tasks" Version="$(AsyncExPackageVersion)" />
127124
</ItemGroup>
128-
129125
<PropertyGroup Condition="'$(TargetFramework)'=='net5.0'">
130126
<AssemblyTitle>Json.NET Tests .NET Standard 2.0</AssemblyTitle>
131127
<ReferringTargetFrameworkForProjectReferences>.NETStandard,Version=v2.0</ReferringTargetFrameworkForProjectReferences>
@@ -146,8 +142,6 @@
146142
<PackageReference Include="xunit" Version="$(XunitPackageVersion)" />
147143
<PackageReference Include="xunit.runner.visualstudio" Version="$(XunitRunnerVisualStudioPackageVersion)" />
148144
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="$(MicrosoftNETTestSdkPackageVersion)" />
149-
<PackageReference Include="Bogus" Version="$(BogusPackageVersion)" />
150-
<PackageReference Include="Nito.AsyncEx.Tasks" Version="$(AsyncExPackageVersion)" />
151145
</ItemGroup>
152146
<PropertyGroup Condition="'$(TargetFramework)'=='netcoreapp3.1'">
153147
<AssemblyTitle>Json.NET Tests .NET Standard 1.3</AssemblyTitle>
@@ -168,10 +162,7 @@
168162
<PackageReference Include="xunit" Version="$(XunitPackageVersion)" />
169163
<PackageReference Include="xunit.runner.visualstudio" Version="$(XunitRunnerVisualStudioPackageVersion)" />
170164
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="$(MicrosoftNETTestSdkPackageVersion)" />
171-
<PackageReference Include="Bogus" Version="$(BogusPackageVersion)" />
172-
<PackageReference Include="Nito.AsyncEx.Tasks" Version="$(AsyncExPackageVersion)" />
173165
</ItemGroup>
174-
175166
<PropertyGroup Condition="'$(TargetFramework)'=='netcoreapp2.1'">
176167
<AssemblyTitle>Json.NET Tests .NET Standard 1.0</AssemblyTitle>
177168
<ReferringTargetFrameworkForProjectReferences>.NETStandard,Version=v1.0</ReferringTargetFrameworkForProjectReferences>

‎Src/Newtonsoft.Json/Linq/JToken.cs

Lines changed: 19 additions & 56 deletions
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,6 @@
4444
using Newtonsoft.Json.Utilities.LinqBridge;
4545
#else
4646
using System.Linq;
47-
using System.Threading;
4847
#endif
4948

5049
namespace Newtonsoft.Json.Linq
@@ -2308,7 +2307,7 @@ int IJsonLineInfo.LinePosition
23082307
/// <returns>A <see cref="JToken"/>, or <c>null</c>.</returns>
23092308
public JToken? SelectToken(string path)
23102309
{
2311-
return SelectToken(path, false);
2310+
return SelectToken(path, settings: null);
23122311
}
23132312

23142313
/// <summary>
@@ -2321,38 +2320,27 @@ int IJsonLineInfo.LinePosition
23212320
/// <returns>A <see cref="JToken"/>.</returns>
23222321
public JToken? SelectToken(string path, bool errorWhenNoMatch)
23232322
{
2324-
JPath p = new JPath(path);
2323+
JsonSelectSettings? settings = errorWhenNoMatch
2324+
? new JsonSelectSettings { ErrorWhenNoMatch = true }
2325+
: null;
23252326

2326-
JToken? token = null;
2327-
foreach (JToken t in p.Evaluate(this, this, errorWhenNoMatch))
2328-
{
2329-
if (token != null)
2330-
{
2331-
throw new JsonException("Path returned multiple tokens.");
2332-
}
2333-
2334-
token = t;
2335-
}
2336-
2337-
return token;
2327+
return SelectToken(path, settings);
23382328
}
23392329

2340-
#if HAVE_REGEX_TIMEOUTS
23412330
/// <summary>
23422331
/// Selects a <see cref="JToken"/> using a JSONPath expression. Selects the token that matches the object path.
23432332
/// </summary>
23442333
/// <param name="path">
23452334
/// A <see cref="String"/> that contains a JSONPath expression.
23462335
/// </param>
2347-
/// <param name="errorWhenNoMatch">A flag to indicate whether an error should be thrown if no tokens are found when evaluating part of the expression.</param>
2348-
/// <param name="singleMatchTimeout">the time after which a single call to regex.ismatch must complete, default is forever</param>
2336+
/// <param name="settings">The <see cref="JsonSelectSettings"/> used to select tokens.</param>
23492337
/// <returns>A <see cref="JToken"/>.</returns>
2350-
public JToken? SelectToken(string path, bool errorWhenNoMatch, TimeSpan? singleMatchTimeout = default)
2338+
public JToken? SelectToken(string path, JsonSelectSettings? settings)
23512339
{
2352-
JPath p = new JPath(path, singleMatchTimeout);
2340+
JPath p = new JPath(path);
23532341

23542342
JToken? token = null;
2355-
foreach (JToken t in p.Evaluate(this, this, errorWhenNoMatch))
2343+
foreach (JToken t in p.Evaluate(this, this, settings))
23562344
{
23572345
if (token != null)
23582346
{
@@ -2364,7 +2352,6 @@ int IJsonLineInfo.LinePosition
23642352

23652353
return token;
23662354
}
2367-
#endif
23682355

23692356
/// <summary>
23702357
/// Selects a collection of elements using a JSONPath expression.
@@ -2375,7 +2362,7 @@ int IJsonLineInfo.LinePosition
23752362
/// <returns>An <see cref="IEnumerable{T}"/> of <see cref="JToken"/> that contains the selected elements.</returns>
23762363
public IEnumerable<JToken> SelectTokens(string path)
23772364
{
2378-
return SelectTokens(path, false);
2365+
return SelectTokens(path, settings: null);
23792366
}
23802367

23812368
/// <summary>
@@ -2388,50 +2375,26 @@ public IEnumerable<JToken> SelectTokens(string path)
23882375
/// <returns>An <see cref="IEnumerable{T}"/> of <see cref="JToken"/> that contains the selected elements.</returns>
23892376
public IEnumerable<JToken> SelectTokens(string path, bool errorWhenNoMatch)
23902377
{
2391-
var p = new JPath(path);
2392-
return p.Evaluate(this, this, errorWhenNoMatch);
2378+
JsonSelectSettings? settings = errorWhenNoMatch
2379+
? new JsonSelectSettings { ErrorWhenNoMatch = true }
2380+
: null;
2381+
2382+
return SelectTokens(path, settings);
23932383
}
23942384

2395-
#if HAVE_REGEX_TIMEOUTS
23962385
/// <summary>
23972386
/// Selects a collection of elements using a JSONPath expression.
23982387
/// </summary>
23992388
/// <param name="path">
24002389
/// A <see cref="String"/> that contains a JSONPath expression.
24012390
/// </param>
2402-
/// <param name="errorWhenNoMatch">A flag to indicate whether an error should be thrown if no tokens are found when evaluating part of the expression.</param>
2403-
/// <param name="singleRegexMatchTimeout">
2404-
/// for every token that matches a jpath, the time a regex is permitted to run
2405-
/// against that token before timeout
2406-
/// </param>
2407-
/// <param name="globalRegexMatchTimeout">
2408-
/// the time this method should wait for the given jpath regex to match all tokens.
2409-
/// worst case expected execution time roughly globalRegexMatchTimeout + Min(singleRegexMatchTimeout, globalRegexMatchTimeout)
2410-
/// </param>
2411-
/// <exception cref="System.Text.RegularExpressions.RegexMatchTimeoutException">if single call timeout exceeded</exception>"
2391+
/// <param name="settings">The <see cref="JsonSelectSettings"/> used to select tokens.</param>
24122392
/// <returns>An <see cref="IEnumerable{T}"/> of <see cref="JToken"/> that contains the selected elements.</returns>
2413-
public IEnumerable<JToken> SelectTokens(string path,
2414-
bool errorWhenNoMatch,
2415-
TimeSpan? singleRegexMatchTimeout = default,
2416-
TimeSpan? globalRegexMatchTimeout = default)
2393+
public IEnumerable<JToken> SelectTokens(string path, JsonSelectSettings? settings)
24172394
{
2418-
var singleTimeout = singleRegexMatchTimeout ?? Timeout.InfiniteTimeSpan;
2419-
var globalTimeout = globalRegexMatchTimeout ?? Timeout.InfiniteTimeSpan;
2420-
if (globalTimeout != Timeout.InfiniteTimeSpan && globalTimeout < singleTimeout)
2421-
{
2422-
singleTimeout = globalTimeout;
2423-
}
2424-
2425-
using var cts = new CancellationTokenSource(globalTimeout);
2426-
var p = new JPath(path, singleTimeout);
2427-
var results = p.Evaluate(this, this, errorWhenNoMatch);
2428-
foreach (var result in results)
2429-
{
2430-
cts.Token.ThrowIfCancellationRequested();
2431-
yield return result;
2432-
}
2395+
var p = new JPath(path);
2396+
return p.Evaluate(this, this, settings);
24332397
}
2434-
#endif
24352398

24362399
#if HAVE_DYNAMIC
24372400
/// <summary>

‎Src/Newtonsoft.Json/Linq/JsonPath/ArrayIndexFilter.cs

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,13 +8,13 @@ internal class ArrayIndexFilter : PathFilter
88
{
99
public int? Index { get; set; }
1010

11-
public override IEnumerable<JToken> ExecuteFilter(JToken root, IEnumerable<JToken> current, bool errorWhenNoMatch)
11+
public override IEnumerable<JToken> ExecuteFilter(JToken root, IEnumerable<JToken> current, JsonSelectSettings? settings)
1212
{
1313
foreach (JToken t in current)
1414
{
1515
if (Index != null)
1616
{
17-
JToken? v = GetTokenIndex(t, errorWhenNoMatch, Index.GetValueOrDefault());
17+
JToken? v = GetTokenIndex(t, settings, Index.GetValueOrDefault());
1818

1919
if (v != null)
2020
{
@@ -32,7 +32,7 @@ public override IEnumerable<JToken> ExecuteFilter(JToken root, IEnumerable<JToke
3232
}
3333
else
3434
{
35-
if (errorWhenNoMatch)
35+
if (settings?.ErrorWhenNoMatch ?? false)
3636
{
3737
throw new JsonException("Index * not valid on {0}.".FormatWith(CultureInfo.InvariantCulture, t.GetType().Name));
3838
}

‎Src/Newtonsoft.Json/Linq/JsonPath/ArrayMultipleIndexFilter.cs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,13 +11,13 @@ public ArrayMultipleIndexFilter(List<int> indexes)
1111
Indexes = indexes;
1212
}
1313

14-
public override IEnumerable<JToken> ExecuteFilter(JToken root, IEnumerable<JToken> current, bool errorWhenNoMatch)
14+
public override IEnumerable<JToken> ExecuteFilter(JToken root, IEnumerable<JToken> current, JsonSelectSettings? settings)
1515
{
1616
foreach (JToken t in current)
1717
{
1818
foreach (int i in Indexes)
1919
{
20-
JToken? v = GetTokenIndex(t, errorWhenNoMatch, i);
20+
JToken? v = GetTokenIndex(t, settings, i);
2121

2222
if (v != null)
2323
{

‎Src/Newtonsoft.Json/Linq/JsonPath/ArraySliceFilter.cs

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ internal class ArraySliceFilter : PathFilter
1111
public int? End { get; set; }
1212
public int? Step { get; set; }
1313

14-
public override IEnumerable<JToken> ExecuteFilter(JToken root, IEnumerable<JToken> current, bool errorWhenNoMatch)
14+
public override IEnumerable<JToken> ExecuteFilter(JToken root, IEnumerable<JToken> current, JsonSelectSettings? settings)
1515
{
1616
if (Step == 0)
1717
{
@@ -56,7 +56,7 @@ public override IEnumerable<JToken> ExecuteFilter(JToken root, IEnumerable<JToke
5656
}
5757
else
5858
{
59-
if (errorWhenNoMatch)
59+
if (settings?.ErrorWhenNoMatch ?? false)
6060
{
6161
throw new JsonException("Array slice of {0} to {1} returned no results.".FormatWith(CultureInfo.InvariantCulture,
6262
Start != null ? Start.GetValueOrDefault().ToString(CultureInfo.InvariantCulture) : "*",
@@ -66,7 +66,7 @@ public override IEnumerable<JToken> ExecuteFilter(JToken root, IEnumerable<JToke
6666
}
6767
else
6868
{
69-
if (errorWhenNoMatch)
69+
if (settings?.ErrorWhenNoMatch ?? false)
7070
{
7171
throw new JsonException("Array slice is not valid on {0}.".FormatWith(CultureInfo.InvariantCulture, t.GetType().Name));
7272
}

‎Src/Newtonsoft.Json/Linq/JsonPath/FieldFilter.cs

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ public FieldFilter(string? name)
1313
Name = name;
1414
}
1515

16-
public override IEnumerable<JToken> ExecuteFilter(JToken root, IEnumerable<JToken> current, bool errorWhenNoMatch)
16+
public override IEnumerable<JToken> ExecuteFilter(JToken root, IEnumerable<JToken> current, JsonSelectSettings? settings)
1717
{
1818
foreach (JToken t in current)
1919
{
@@ -27,7 +27,7 @@ public override IEnumerable<JToken> ExecuteFilter(JToken root, IEnumerable<JToke
2727
{
2828
yield return v;
2929
}
30-
else if (errorWhenNoMatch)
30+
else if (settings?.ErrorWhenNoMatch ?? false)
3131
{
3232
throw new JsonException("Property '{0}' does not exist on JObject.".FormatWith(CultureInfo.InvariantCulture, Name));
3333
}
@@ -42,7 +42,7 @@ public override IEnumerable<JToken> ExecuteFilter(JToken root, IEnumerable<JToke
4242
}
4343
else
4444
{
45-
if (errorWhenNoMatch)
45+
if (settings?.ErrorWhenNoMatch ?? false)
4646
{
4747
throw new JsonException("Property '{0}' not valid on {1}.".FormatWith(CultureInfo.InvariantCulture, Name ?? "*", t.GetType().Name));
4848
}

0 commit comments

Comments
 (0)