Skip to content

Commit e3f7c99

Browse files
kaby76claude
kaby76
and
claude
authored
[CI] Add performance information for all grammars in repo. (#4919)
* Update testing templates, and add initial version of perf test table script. * [test-tps] Add N-run averaging with mean ± SEM markdown table - Run each grammar's Test.exe N=3 times (configurable) and compute mean ± SEM for PT, OT, TT, TPS, Post-warmup TPS, and Post-warmup speed up; output results as a markdown table at the end of the run - Add a silent warmup run before the N-loop to absorb Windows AV pre-scan overhead, which inflated OT variance on the first real run - Fix -g flag: explicitly specified grammars now bypass the filter=all discovery scan instead of being overwritten by it - Remove stale debug check ("what the f." block) Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com> * Add performance.md. * Add sortable HTML view of performance table Adds performance.html (generated from performance.md) with clickable column headers that sort ascending/descending. Also includes the conversion script md_to_sortable_html.py. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com> * Add temporary link to sortable performance table (htmlpreview) Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com> * Add column key to performance.md and performance.html Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com> * Fix ± encoding corruption in performance.md and performance.html Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com> * Render Markdown inline formatting in HTML key table Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com> * Left-justify Description column in HTML key table Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com> * Use inline styles to left-justify key table cells Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com> * Fix punctuation in README.md * Change link to performance table with final value in repo. * Add antlr/grammars-v4 links to grammar names in performance table Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com> --------- Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com>
1 parent 7abc3f1 commit e3f7c99

18 files changed

Lines changed: 1806 additions & 42 deletions

File tree

‎.github/dependabot.yml‎

Lines changed: 0 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,5 @@
11
version: 2
22
updates:
3-
# dotnet tools
4-
- package-ecosystem: "nuget"
5-
directory: "/"
6-
schedule:
7-
interval: "daily"
8-
labels:
9-
- "infrastructure"
10-
- "dependencies"
11-
ignore:
12-
- dependency-name: "tr*"
133
# Java
144
- package-ecosystem: "maven"
155
directory: "/"

‎README.md‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,13 @@ Weekly dev build: [![Weekly Dev](https://github.com/antlr/grammars-v4/actions/wo
44

55
# Grammars-v4
66

7-
This repository is a collection of formal grammars written for [ANTLR v4](https://github.com/antlr/antlr4)
7+
This repository is a collection of formal grammars written for [ANTLR v4](https://github.com/antlr/antlr4).
88

9-
The root directory name is the all-lowercase name of the language or file format parsed by the grammar. For example, java, cpp, csharp, c, etc...
9+
The root directory name is the all-lowercase name of the language or file format parsed by the grammar. For example, java, cpp, csharp, c, etc.
10+
11+
## Performance
12+
13+
[Grammar performance table](https://htmlpreview.github.io/?https://github.com/antlr/grammars-v4/blob/performance.html) (sortable by column header).
1014

1115
## FAQ
1216

‎_scripts/templates/Antlr4cs/st.Test.cs‎

Lines changed: 30 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,10 @@ public class Program
3535
static int string_instance = 0;
3636
static string prefix = "";
3737
static bool quiet = false;
38+
static long total_tokens = 0;
39+
static double total_parse_seconds = 0;
40+
static long first_file_tokens = 0;
41+
static double first_file_parse_seconds = 0;
3842

3943
static void Main(string[] args)
4044
{
@@ -125,7 +129,22 @@ static void Main(string[] args)
125129
DateTime after = DateTime.Now;
126130
if (!quiet)
127131
{
128-
System.Console.Error.WriteLine(prefix + "Total Time: " + (after - before).TotalSeconds);
132+
var overall_seconds = (after - before).TotalSeconds;
133+
var warm_tokens = total_tokens - first_file_tokens;
134+
var warm_seconds = total_parse_seconds - first_file_parse_seconds;
135+
var warm_tps = (inputs.Count() > 1 && warm_seconds > 0)
136+
? ((long)(warm_tokens / warm_seconds)).ToString()
137+
: "n.a.";
138+
var first_tps = first_file_parse_seconds > 0 ? (first_file_tokens / first_file_parse_seconds) : 0;
139+
var speedup = (inputs.Count() > 1 && warm_seconds > 0 && first_tps > 0)
140+
? ((warm_tokens / warm_seconds) / first_tps).ToString("F2")
141+
: "n.a.";
142+
System.Console.Error.WriteLine(prefix + "PT: " + total_parse_seconds);
143+
System.Console.Error.WriteLine(prefix + "OT: " + (overall_seconds - total_parse_seconds));
144+
System.Console.Error.WriteLine(prefix + "TT: " + overall_seconds);
145+
System.Console.Error.WriteLine(prefix + "TPS: " + (long)(total_tokens / total_parse_seconds));
146+
System.Console.Error.WriteLine(prefix + "Post-warmup TPS: " + warm_tps);
147+
System.Console.Error.WriteLine(prefix + "Post-warmup speed up: " + speedup);
129148
}
130149
}
131150
Environment.ExitCode = exit_code;
@@ -200,6 +219,15 @@ static void DoParse(ICharStream str, string input_name, int row_number)
200219
DateTime before = DateTime.Now;
201220
var tree = parser.<start_symbol>();
202221
DateTime after = DateTime.Now;
222+
var parse_seconds = (after - before).TotalSeconds;
223+
total_parse_seconds += parse_seconds;
224+
var token_count = tokens.Size;
225+
total_tokens += token_count;
226+
if (row_number == 0)
227+
{
228+
first_file_tokens = token_count;
229+
first_file_parse_seconds = parse_seconds;
230+
}
203231
var result = "";
204232
if (parser.NumberOfSyntaxErrors > 0)
205233
{
@@ -222,7 +250,7 @@ static void DoParse(ICharStream str, string input_name, int row_number)
222250
}
223251
if (!quiet)
224252
{
225-
System.Console.Error.WriteLine(prefix + "Antlr4cs " + row_number + " " + input_name + " " + result + " " + (after - before).TotalSeconds);
253+
System.Console.Error.WriteLine(prefix + "Antlr4cs " + row_number + " " + input_name + " " + result + " " + parse_seconds + " s " + token_count + " tokens " + (long)(token_count / parse_seconds) + " tps");
226254
}
227255
if (tee) output.Close();
228256
}

‎_scripts/templates/Antlr4ng/st.Test.ts‎

Lines changed: 29 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,10 @@ var enc = '<file_encoding>';
5757
var binary = <binary>;
5858
var string_instance = 0;
5959
var prefix = '';
60+
var total_tokens = 0;
61+
var total_parse_seconds = 0;
62+
var first_file_tokens = 0;
63+
var first_file_parse_seconds = 0;
6064
var inputs: string[] = [];
6165
var is_fns: boolean[] = [];
6266

@@ -128,7 +132,23 @@ function main() {
128132
}
129133
timer.stop();
130134
var t = timer.time().m * 60 + timer.time().s + timer.time().ms / 1000;
131-
if (!quiet) console.error(prefix + 'Total Time: ' + t);
135+
if (!quiet) {
136+
var warm_tokens = total_tokens - first_file_tokens;
137+
var warm_seconds = total_parse_seconds - first_file_parse_seconds;
138+
var warm_tps = (inputs.length > 1 && warm_seconds > 0)
139+
? Math.round(warm_tokens / warm_seconds).toString()
140+
: 'n.a.';
141+
var first_tps = first_file_parse_seconds > 0 ? (first_file_tokens / first_file_parse_seconds) : 0;
142+
var speedup = (inputs.length > 1 && warm_seconds > 0 && first_tps > 0)
143+
? ((warm_tokens / warm_seconds) / first_tps).toFixed(2)
144+
: 'n.a.';
145+
console.error(prefix + 'PT: ' + total_parse_seconds);
146+
console.error(prefix + 'OT: ' + (t - total_parse_seconds));
147+
console.error(prefix + 'TT: ' + t);
148+
console.error(prefix + 'TPS: ' + Math.round(total_tokens / total_parse_seconds));
149+
console.error(prefix + 'Post-warmup TPS: ' + warm_tps);
150+
console.error(prefix + 'Post-warmup speed up: ' + speedup);
151+
}
132152
}
133153
process.exitCode = error_code;
134154
}
@@ -189,6 +209,8 @@ function DoParse(str: CharStream, input_name: string, row_number: number) {
189209
timer.start();
190210
const tree = parser.<start_symbol>();
191211
timer.stop();
212+
var token_count = tokens.size;
213+
total_tokens += token_count;
192214
var result = "";
193215
if (listener_parser.had_error || listener_lexer.had_error) {
194216
result = 'fail';
@@ -198,6 +220,11 @@ function DoParse(str: CharStream, input_name: string, row_number: number) {
198220
result = 'success';
199221
}
200222
var t = timer.time().m * 60 + timer.time().s + timer.time().ms / 1000;
223+
total_parse_seconds += t;
224+
if (row_number == 0) {
225+
first_file_tokens = token_count;
226+
first_file_parse_seconds = t;
227+
}
201228
if (show_tree) {
202229
if (tee) {
203230
writeFileSync(input_name + ".tree", tree.toStringTree(parser.ruleNames, parser));
@@ -206,7 +233,7 @@ function DoParse(str: CharStream, input_name: string, row_number: number) {
206233
}
207234
}
208235
if (!quiet) {
209-
console.error(prefix + 'TypeScript ' + row_number + ' ' + input_name + ' ' + result + ' ' + t);
236+
console.error(prefix + 'TypeScript ' + row_number + ' ' + input_name + ' ' + result + ' ' + t + ' s ' + token_count + ' tokens ' + Math.round(token_count / t) + ' tps');
210237
}
211238
if (tee) {
212239
closeSync(output);

‎_scripts/templates/CSharp/st.Test.cs‎

Lines changed: 35 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -176,6 +176,10 @@ public static IParseTree Parse(string input)
176176
static bool show_tokens = false;
177177
static bool show_token_count = false;
178178
static long total_count = 0;
179+
static long total_tokens = 0;
180+
static double total_parse_seconds = 0;
181+
static long first_file_tokens = 0;
182+
static double first_file_parse_seconds = 0;
179183
static bool show_trace = false;
180184
static bool show_tree = false;
181185
static bool old = false;
@@ -328,7 +332,25 @@ static void Main(string[] args)
328332
ParseString(inputs[f], f);
329333
}
330334
DateTime after = DateTime.Now;
331-
if (!quiet) System.Console.Error.WriteLine(prefix + "Total Time: " + (after - before).TotalSeconds);
335+
if (!quiet)
336+
{
337+
var overall_seconds = (after - before).TotalSeconds;
338+
var warm_tokens = total_tokens - first_file_tokens;
339+
var warm_seconds = total_parse_seconds - first_file_parse_seconds;
340+
var warm_tps = (inputs.Count() > 1 && warm_seconds > 0)
341+
? ((long)(warm_tokens / warm_seconds)).ToString()
342+
: "n.a.";
343+
var first_tps = first_file_parse_seconds > 0 ? (first_file_tokens / first_file_parse_seconds) : 0;
344+
var speedup = (inputs.Count() > 1 && warm_seconds > 0 && first_tps > 0)
345+
? ((warm_tokens / warm_seconds) / first_tps).ToString("F2")
346+
: "n.a.";
347+
System.Console.Error.WriteLine(prefix + "PT: " + total_parse_seconds);
348+
System.Console.Error.WriteLine(prefix + "OT: " + (overall_seconds - total_parse_seconds));
349+
System.Console.Error.WriteLine(prefix + "TT: " + overall_seconds);
350+
System.Console.Error.WriteLine(prefix + "TPS: " + (long)(total_tokens / total_parse_seconds));
351+
System.Console.Error.WriteLine(prefix + "Post-warmup TPS: " + warm_tps);
352+
System.Console.Error.WriteLine(prefix + "Post-warmup speed up: " + speedup);
353+
}
332354
if (show_token_count) System.Console.Error.WriteLine("TC: " + total_count);
333355
}
334356
Environment.ExitCode = exit_code;
@@ -359,6 +381,8 @@ static void ParseFilename(string input, int row_number)
359381
str = new Antlr4.Runtime.AntlrInputStream(fs);
360382
}
361383
else if (file_encoding == null || file_encoding == "")
384+
str = CharStreams.fromPath(input);
385+
else if (file_encoding == "detect")
362386
{
363387
var detected = CharsetDetector.DetectFromFile(input);
364388
var enc = detected.Detected?.Encoding ?? Encoding.UTF8;
@@ -431,6 +455,15 @@ static void DoParse(ICharStream str, string input_name, int row_number)
431455
DateTime before = DateTime.Now;
432456
var tree = parser.<start_symbol>();
433457
DateTime after = DateTime.Now;
458+
var parse_seconds = (after - before).TotalSeconds;
459+
total_parse_seconds += parse_seconds;
460+
var token_count = tokens.Size;
461+
total_tokens += token_count;
462+
if (row_number == 0)
463+
{
464+
first_file_tokens = token_count;
465+
first_file_parse_seconds = parse_seconds;
466+
}
434467
var result = "";
435468
if (listener_lexer.had_error || listener_parser.had_error)
436469
{
@@ -492,7 +525,7 @@ static void DoParse(ICharStream str, string input_name, int row_number)
492525
}
493526
if (!quiet)
494527
{
495-
System.Console.Error.WriteLine(prefix + "CSharp " + row_number + " " + input_name + " " + result + " " + (after - before).TotalSeconds);
528+
System.Console.Error.WriteLine(prefix + "CSharp " + row_number + " " + input_name + " " + result + " " + parse_seconds + " s " + token_count + " tokens " + (long)(token_count / parse_seconds) + " tps");
496529
}
497530

498531
if (earley) {

‎_scripts/templates/Cpp/st.Test.cpp‎

Lines changed: 33 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,10 @@ int string_instance = 0;
5353
std::string prefix;
5454
bool quiet = false;
5555
std::string file_encoding = "<file_encoding>";
56+
long total_tokens = 0;
57+
double total_parse_seconds = 0;
58+
long first_file_tokens = 0;
59+
double first_file_parse_seconds = 0;
5660

5761
void DoParse(antlr4::CharStream* str, std::string input_name, int row_number)
5862
{
@@ -89,7 +93,15 @@ void DoParse(antlr4::CharStream* str, std::string input_name, int row_number)
8993
auto before = std::chrono::steady_clock::now();
9094
auto* tree = parser-><start_symbol>();
9195
auto after = std::chrono::steady_clock::now();
96+
long token_count = (long)tokens->size();
97+
total_tokens += token_count;
9298
auto duration = std::chrono::duration_cast\<std::chrono::microseconds>(after - before);
99+
double parse_seconds = duration.count() / 1000000.0;
100+
total_parse_seconds += parse_seconds;
101+
if (row_number == 0) {
102+
first_file_tokens = token_count;
103+
first_file_parse_seconds = parse_seconds;
104+
}
93105
std::string result;
94106
if (listener_parser->had_error || listener_lexer->had_error)
95107
{
@@ -120,7 +132,7 @@ void DoParse(antlr4::CharStream* str, std::string input_name, int row_number)
120132
}
121133
if (!quiet)
122134
{
123-
std::cerr \<\< prefix \<\< "Cpp " \<\< row_number \<\< " " \<\< input_name \<\< " " \<\< result \<\< " " \<\< formatDurationSeconds(duration.count()) \<\< std::endl;
135+
std::cerr \<\< prefix \<\< "Cpp " \<\< row_number \<\< " " \<\< input_name \<\< " " \<\< result \<\< " " \<\< parse_seconds \<\< " s " \<\< token_count \<\< " tokens " \<\< (long)(token_count / parse_seconds) \<\< " tps" \<\< std::endl;
124136
}
125137
if (tee)
126138
{
@@ -241,7 +253,26 @@ int TryParse(std::vector\<std::string>& args)
241253
}
242254
auto after = std::chrono::steady_clock::now();
243255
auto duration = std::chrono::duration_cast\<std::chrono::microseconds>(after - before);
244-
if (! quiet) std::cerr \<\< prefix \<\< "Total Time: " \<\< formatDurationSeconds(duration.count()) \<\< std::endl;
256+
if (!quiet) {
257+
double overall_seconds = duration.count() / 1000000.0;
258+
long warm_tokens = total_tokens - first_file_tokens;
259+
double warm_seconds = total_parse_seconds - first_file_parse_seconds;
260+
std::string warm_tps = (inputs.size() > 1 && warm_seconds > 0)
261+
? std::to_string((long)(warm_tokens / warm_seconds))
262+
: "n.a.";
263+
double first_tps = first_file_parse_seconds > 0 ? (first_file_tokens / first_file_parse_seconds) : 0;
264+
std::ostringstream speedup_ss;
265+
if (inputs.size() > 1 && warm_seconds > 0 && first_tps > 0)
266+
speedup_ss \<\< std::fixed \<\< std::setprecision(2) \<\< ((warm_tokens / warm_seconds) / first_tps);
267+
else
268+
speedup_ss \<\< "n.a.";
269+
std::cerr \<\< prefix \<\< "PT: " \<\< total_parse_seconds \<\< std::endl;
270+
std::cerr \<\< prefix \<\< "OT: " \<\< (overall_seconds - total_parse_seconds) \<\< std::endl;
271+
std::cerr \<\< prefix \<\< "TT: " \<\< overall_seconds \<\< std::endl;
272+
std::cerr \<\< prefix \<\< "TPS: " \<\< (long)(total_tokens / total_parse_seconds) \<\< std::endl;
273+
std::cerr \<\< prefix \<\< "Post-warmup TPS: " \<\< warm_tps \<\< std::endl;
274+
std::cerr \<\< prefix \<\< "Post-warmup speed up: " \<\< speedup_ss.str() \<\< std::endl;
275+
}
245276
}
246277
return error_code;
247278
}

‎_scripts/templates/Dart/st.Test.dart‎

Lines changed: 29 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,10 @@ var error_code = 0;
6060
var string_instance = 0;
6161
var prefix = "";
6262
var quiet = false;
63+
var total_tokens = 0;
64+
var total_parse_seconds = 0.0;
65+
var first_file_tokens = 0;
66+
var first_file_parse_seconds = 0.0;
6367

6468
void main(List\<String> args) async {
6569
// Set command-line args before anything else.
@@ -136,7 +140,23 @@ void main(List\<String> args) async {
136140
}
137141
s.stop();
138142
var et = s.elapsedMilliseconds / 1000.0;
139-
if (!quiet) stderr.writeln(prefix + "Total Time: " + et.toString());
143+
if (!quiet) {
144+
var warm_tokens = total_tokens - first_file_tokens;
145+
var warm_seconds = total_parse_seconds - first_file_parse_seconds;
146+
var warm_tps = (inputs.length > 1 && warm_seconds > 0)
147+
? (warm_tokens / warm_seconds).round().toString()
148+
: "n.a.";
149+
var first_tps = first_file_parse_seconds > 0 ? (first_file_tokens / first_file_parse_seconds) : 0.0;
150+
var speedup = (inputs.length > 1 && warm_seconds > 0 && first_tps > 0)
151+
? ((warm_tokens / warm_seconds) / first_tps).toStringAsFixed(2)
152+
: "n.a.";
153+
stderr.writeln(prefix + "PT: " + total_parse_seconds.toString());
154+
stderr.writeln(prefix + "OT: " + (et - total_parse_seconds).toString());
155+
stderr.writeln(prefix + "TT: " + et.toString());
156+
stderr.writeln(prefix + "TPS: " + (total_parse_seconds > 0 ? (total_tokens / total_parse_seconds).round().toString() : "0"));
157+
stderr.writeln(prefix + "Post-warmup TPS: " + warm_tps);
158+
stderr.writeln(prefix + "Post-warmup speed up: " + speedup);
159+
}
140160
}
141161
exit(error_code);
142162
}
@@ -220,7 +240,14 @@ Future\<void> DoParse(CharStream str, String input_name, int row_number) async
220240
s.start();
221241
var tree = parser.<start_symbol>();
222242
s.stop();
243+
var token_count = tokens.size;
244+
total_tokens += token_count;
223245
var et = s.elapsedMilliseconds / 1000.0;
246+
total_parse_seconds += et;
247+
if (row_number == 0) {
248+
first_file_tokens = token_count;
249+
first_file_parse_seconds = et;
250+
}
224251
var result = "";
225252
if (parser.numberOfSyntaxErrors > 0)
226253
{
@@ -249,7 +276,7 @@ Future\<void> DoParse(CharStream str, String input_name, int row_number) async
249276
}
250277
if (!quiet)
251278
{
252-
stderr.writeln(prefix + "Dart " + row_number.toString() + " " + input_name + " " + result + " " + et.toString());
279+
stderr.writeln(prefix + "Dart " + row_number.toString() + " " + input_name + " " + result + " " + et.toString() + " s " + token_count.toString() + " tokens " + (et > 0 ? (token_count / et).round().toString() : "0") + " tps");
253280
}
254281
if (tee)
255282
{

0 commit comments

Comments
 (0)