From a95d651772014afe4a108edb65594574a3b4a9e1 Mon Sep 17 00:00:00 2001 From: David Morgan Date: Wed, 5 Jun 2024 12:24:00 +0200 Subject: [PATCH] [macros] JSON benchmarks, JsonBuffer experiment. --- working/macros/dart_model/README.md | 6 +- .../testing/json_benchmark/bin/main.dart | 3 + .../json_benchmark/lib/json_benchmark.dart | 97 ++++++ .../json_benchmark/lib/json_buffer.dart | 296 ++++++++++++++++++ .../lib/json_buffer_subject.dart | 99 ++++++ .../json_benchmark/lib/json_subject.dart | 69 ++++ .../testing/json_benchmark/lib/subject.dart | 14 + .../testing/json_benchmark/pubspec.yaml | 8 + .../json_benchmark/test/json_buffer_test.dart | 67 ++++ working/macros/dart_model/testing/presubmit | 3 +- working/macros/dart_model/testing/pub_get | 3 +- 11 files changed, 662 insertions(+), 3 deletions(-) create mode 100644 working/macros/dart_model/testing/json_benchmark/bin/main.dart create mode 100644 working/macros/dart_model/testing/json_benchmark/lib/json_benchmark.dart create mode 100644 working/macros/dart_model/testing/json_benchmark/lib/json_buffer.dart create mode 100644 working/macros/dart_model/testing/json_benchmark/lib/json_buffer_subject.dart create mode 100644 working/macros/dart_model/testing/json_benchmark/lib/json_subject.dart create mode 100644 working/macros/dart_model/testing/json_benchmark/lib/subject.dart create mode 100644 working/macros/dart_model/testing/json_benchmark/pubspec.yaml create mode 100644 working/macros/dart_model/testing/json_benchmark/test/json_buffer_test.dart diff --git a/working/macros/dart_model/README.md b/working/macros/dart_model/README.md index 306b748b2c..f8f73eb1b8 100644 --- a/working/macros/dart_model/README.md +++ b/working/macros/dart_model/README.md @@ -21,7 +21,7 @@ as a library;\ The "macros" referred to in this exploration are independent of the in-progress macro implementation, hence the "scare quotes". -## Benchmarks +## End to End Benchmarks `testing/benchmark` is a tool to assist in benchmarking, it creates codebases of the specified size and codegen strategy. @@ -65,3 +65,7 @@ $ dart bin/main.dart # files to see how the analyzer responds; you can watch the macro host terminal # to see when it is rewriting augmentation files. ``` + +## Serialization Benchmarks + +`testing/json_benchmark` is benchmarking related to JSON serialization. diff --git a/working/macros/dart_model/testing/json_benchmark/bin/main.dart b/working/macros/dart_model/testing/json_benchmark/bin/main.dart new file mode 100644 index 0000000000..a87d5caf16 --- /dev/null +++ b/working/macros/dart_model/testing/json_benchmark/bin/main.dart @@ -0,0 +1,3 @@ +import 'package:json_benchmark/json_benchmark.dart'; + +Future main() async => JsonBenchmark().run(); diff --git a/working/macros/dart_model/testing/json_benchmark/lib/json_benchmark.dart b/working/macros/dart_model/testing/json_benchmark/lib/json_benchmark.dart new file mode 100644 index 0000000000..a7cdf9dd2d --- /dev/null +++ b/working/macros/dart_model/testing/json_benchmark/lib/json_benchmark.dart @@ -0,0 +1,97 @@ +import 'dart:io'; + +import 'json_buffer_subject.dart'; +import 'json_subject.dart'; + +class JsonBenchmark { + Future run() async { + final jsonSubject = JsonSubject(); + + print('Subject,Scenario,Data size/bytes,Time per/ms'); + for (final subject in [ + JsonBufferSubject(), + JsonBufferSubject(), + JsonBufferSubject(), + JsonBufferSubject(), + JsonBufferSubject(), /*JsonSubject()*/ + ]) { + for (final size in [256]) { + final neutralData = jsonSubject.createData(libraryCount: size); + final subjectData = subject.deepCopyIn(neutralData); + final byteData = subject.serialize(subjectData); + final byteLength = byteData.length; + + await benchmark(subject.name, 'create', byteLength, + () => subject.createData(libraryCount: size)); + await benchmark(subject.name, 'deepCopyIn', byteLength, + () => subject.deepCopyIn(neutralData)); + await benchmark(subject.name, 'serialize', byteLength, + () => subject.serialize(subjectData)); + await benchmark(subject.name, 'writeSync', byteLength, + () => File('/tmp/benchmark').writeAsBytesSync(byteData)); + await benchmark( + subject.name, + 'copySerializeWrite', + byteLength, + () => File('/tmp/benchmark').writeAsBytesSync( + subject.serialize(subject.deepCopyIn(neutralData)))); + await benchmark( + subject.name, + 'createSerializeWrite', + byteLength, + () => File('/tmp/benchmark').writeAsBytesSync( + subject.serialize(subject.createData(libraryCount: size)))); + + await benchmark( + subject.name, 'process', byteLength, () => process(subjectData)); + await benchmark(subject.name, 'deepCopyOut', byteLength, + () => subject.deepCopyOut(subjectData)); + await benchmark(subject.name, 'readSync', byteLength, + () => File('/tmp/benchmark').readAsBytesSync()); + await benchmark(subject.name, 'deserialize', byteLength, + () => subject.deserialize(byteData)); + await benchmark( + subject.name, + 'readDeserializeCopy', + byteLength, + () => subject.deepCopyOut( + subject.deserialize(File('/tmp/benchmark').readAsBytesSync()))); + await benchmark( + subject.name, + 'readDeserializeProcess', + byteLength, + () => process( + subject.deserialize(File('/tmp/benchmark').readAsBytesSync()))); + } + } + } + + int process(Map data) { + var result = 0; + for (final entry in data.entries) { + final key = entry.key; + result ^= key.hashCode; + var value = entry.value; + if (value is Map) { + result ^= process(value); + } else { + result ^= value.hashCode; + } + } + return result; + } + + Future benchmark(String subjectName, String scenarioName, int length, + Function subject) async { + final repetitions = 100; + for (var i = 0; i != repetitions; ++i) { + subject(); + } + final stopwatch = Stopwatch()..start(); + for (var i = 0; i != repetitions; ++i) { + subject(); + } + final elapsed = stopwatch.elapsedMilliseconds; + print('$subjectName,$scenarioName,$length,${elapsed / repetitions}'); + } +} diff --git a/working/macros/dart_model/testing/json_benchmark/lib/json_buffer.dart b/working/macros/dart_model/testing/json_benchmark/lib/json_buffer.dart new file mode 100644 index 0000000000..52397a7a64 --- /dev/null +++ b/working/macros/dart_model/testing/json_benchmark/lib/json_buffer.dart @@ -0,0 +1,296 @@ +import 'dart:collection'; +import 'dart:convert'; +import 'dart:typed_data'; + +typedef Pointer = int; + +enum Type { + string, + bool, + map, +} + +final typeSize = 1; +final intSize = 4; + +class JsonBuffer { + List? _keys; + Object? Function(String)? _function; + final Map _seenStrings = {}; + final Map _decodedStrings = {}; + + Uint8List _buffer = Uint8List(1024); + int _nextFree = 0; + + // TODO: add a keyFunction like the Map constructor. + JsonBuffer( + {required Iterable keys, + required Object? Function(String) function}) + : _keys = keys.toList(), + _function = function; + + JsonBuffer.deserialize(this._buffer) + : _keys = null, + _function = null, + _nextFree = _buffer.length; + + Uint8List serialize() { + _evaluate(); + return _buffer.sublist(0, _nextFree); + } + + void _evaluate() { + if (_keys != null) { + _add(_keys!, _function!); + _keys = null; + _function = null; + } + } + + void _add(Iterable keys, Object? Function(String) function) { + final keysList = keys.toList(); + final length = keysList.length; + final start = _nextFree; + _reserve(length * intSize * 2 + intSize); + _writeInt(start, intSize, length); + for (var i = 0; i != length; ++i) { + final key = keysList[i]; + _writeInt(start + intSize + i * intSize * 2, intSize, _addString(key)); + final value = function(key); + _writeInt(start + intSize + i * intSize * 2 + intSize, intSize, + _addValue(value)); + } + } + + void _reserve(int bytes) { + _nextFree += bytes; + while (_nextFree > _buffer.length) { + _expand(); + } + } + + void _expand() { + final oldBuffer = _buffer; + _buffer = Uint8List(_buffer.length * 2); + _buffer.setRange(0, oldBuffer.length, oldBuffer); + } + + Pointer _addValue(Object? value) { + final start = _nextFree; + if (value is String) { + _reserve(typeSize); + _buffer[start] = Type.string.index; + _reserve(intSize); + _writeInt(start + 1, intSize, _addString(value)); + return start; + } else if (value is bool) { + _reserve(typeSize); + _buffer[start] = Type.bool.index; + _addBool(value); + return start; + } else if (value is JsonBuffer) { + _reserve(typeSize); + _buffer[start] = Type.map.index; + _add(value._keys!, value._function!); + return start; + } else { + throw UnsupportedError('Unsupported value type: ${value.runtimeType}'); + } + } + + Pointer _addString(String value) { + final maybeResult = _seenStrings[value]; + if (maybeResult != null) return maybeResult; + final start = _nextFree; + final bytes = utf8.encode(value); + final length = bytes.length; + _reserve(intSize + length); + _writeInt(start, intSize, length); + _buffer.setRange(start + intSize, start + intSize + length, bytes); + _seenStrings[value] = start; + return start; + } + + Pointer _addBool(bool value) { + final start = _nextFree; + _reserve(1); + _buffer[start] = value ? 1 : 0; + return start; + } + + void _writeInt(Pointer pointer, int intSize, int value) { + if (intSize == 1) { + _buffer[pointer] = value; + } else if (intSize == 2) { + _buffer[pointer] = value & 0xff; + _buffer[pointer + 1] = (value >> 8) & 0xff; + } else if (intSize == 3) { + _buffer[pointer] = value & 0xff; + _buffer[pointer + 1] = (value >> 8) & 0xff; + _buffer[pointer + 2] = (value >> 16) & 0xff; + } else if (intSize == 4) { + _buffer[pointer] = value & 0xff; + _buffer[pointer + 1] = (value >> 8) & 0xff; + _buffer[pointer + 2] = (value >> 16) & 0xff; + _buffer[pointer + 3] = (value >> 24) & 0xff; + } else { + throw UnsupportedError('Integer size: $intSize'); + } + } + + int _readInt(Pointer pointer, int intSize) { + if (intSize == 1) { + return _buffer[pointer]; + } else if (intSize == 2) { + return _buffer[pointer] + (_buffer[pointer + 1] << 8); + } else if (intSize == 3) { + return _buffer[pointer] + + (_buffer[pointer + 1] << 8) + + (_buffer[pointer + 2] << 16); + } else if (intSize == 4) { + return _buffer[pointer] + + (_buffer[pointer + 1] << 8) + + (_buffer[pointer + 2] << 16) + + (_buffer[pointer + 3] << 24); + } else { + throw UnsupportedError('Integer size: $intSize'); + } + } + + Object? _readValue(Pointer pointer) { + final type = Type.values[_buffer[pointer]]; + switch (type) { + case Type.string: + return _readString(_readPointer(pointer + typeSize)); + case Type.bool: + return _readBool(pointer + typeSize); + case Type.map: + return JsonBufferMap._(this, pointer + typeSize); + } + } + + Pointer _readPointer(Pointer pointer) { + return _readInt(pointer, intSize); + } + + String _readString(Pointer pointer) { + final maybeResult = _decodedStrings[pointer]; + if (maybeResult != null) return maybeResult; + final length = _readInt(pointer, intSize); + return _decodedStrings[pointer] ??= utf8 + .decode(_buffer.sublist(pointer + intSize, pointer + intSize + length)); + } + + bool _readBool(Pointer pointer) { + final value = _buffer[pointer]; + if (value == 1) return true; + if (value == 0) return false; + throw StateError('Unexpcted bool value: $value'); + } + + late final Map asMap = _createMap(); + + Map _createMap() { + _evaluate(); + return JsonBufferMap._(this, 0); + } + + @override + String toString() => _buffer.toString(); +} + +class JsonBufferMap + with MapMixin + implements Map { + final JsonBuffer _buffer; + final Pointer _pointer; + + JsonBufferMap._(this._buffer, this._pointer); + + Uint8List serialize() => _buffer.serialize(); + + @override + Object? operator [](Object? key) { + final iterator = entries.iterator as JsonBufferMapEntryIterator; + while (iterator.moveNext()) { + if (iterator.current.key == key) return iterator.current.value; + } + return null; + } + + @override + void operator []=(String key, Object? value) { + throw UnsupportedError('JsonBufferMap is readonly.'); + } + + @override + void clear() { + throw UnsupportedError('JsonBufferMap is readonly.'); + } + + @override + late Iterable keys = + JsonBufferMapEntryIterable(_buffer, _pointer, readValues: false) + .map((e) => e.key); + + @override + late Iterable values = + JsonBufferMapEntryIterable(_buffer, _pointer, readKeys: false) + .map((e) => e.value); + + @override + late Iterable> entries = + JsonBufferMapEntryIterable(_buffer, _pointer); + + @override + Object? remove(Object? key) { + throw UnsupportedError('JsonBufferMap is readonly.'); + } +} + +class JsonBufferMapEntryIterable + with IterableMixin> + implements Iterable> { + final JsonBuffer _buffer; + final Pointer _pointer; + final bool readKeys; + final bool readValues; + + JsonBufferMapEntryIterable(this._buffer, this._pointer, + {this.readKeys = true, this.readValues = true}); + + @override + Iterator> get iterator => + JsonBufferMapEntryIterator(_buffer, _pointer); +} + +class JsonBufferMapEntryIterator + implements Iterator> { + final JsonBuffer _buffer; + Pointer _pointer; + final Pointer _last; + final bool readKeys; + final bool readValues; + + JsonBufferMapEntryIterator(this._buffer, Pointer pointer, + {this.readKeys = true, this.readValues = true}) + : _last = pointer + + intSize + + _buffer._readInt(pointer, intSize) * 2 * intSize, + _pointer = pointer - intSize; + + @override + MapEntry get current => MapEntry( + readKeys ? _buffer._readString(_buffer._readPointer(_pointer)) : '', + readValues + ? _buffer._readValue(_buffer._readPointer(_pointer + intSize)) + : null); + + @override + bool moveNext() { + if (_pointer == _last) return false; + if (_pointer > _last) throw StateError('Moved past _last!'); + _pointer += intSize * 2; + return _pointer != _last; + } +} diff --git a/working/macros/dart_model/testing/json_benchmark/lib/json_buffer_subject.dart b/working/macros/dart_model/testing/json_benchmark/lib/json_buffer_subject.dart new file mode 100644 index 0000000000..c431697233 --- /dev/null +++ b/working/macros/dart_model/testing/json_benchmark/lib/json_buffer_subject.dart @@ -0,0 +1,99 @@ +import 'dart:typed_data'; + +import 'json_buffer.dart'; +import 'subject.dart'; + +class JsonBufferSubject implements Subject { + @override + String get name => 'JsonBuffer'; + + @override + Map createData({required int libraryCount}) { + final buffer = JsonBuffer( + keys: [ + for (var i = 0; i != libraryCount; ++i) + 'package:json_benchmark/library$i.dart' + ], + function: (_) => _createLibrary(classCount: 10), + ); + return buffer.asMap; + } + + @override + Map deepCopyIn(Map data) { + return _deepCopyIn(data).asMap; + } + + JsonBuffer _deepCopyIn(Map data) { + return JsonBuffer( + keys: data.keys, + function: (key) { + final value = data[key]; + if (value is Map) { + return _deepCopyIn(value); + } else { + return value; + } + }); + } + + @override + Map deepCopyOut(Map data) { + final result = {}; + for (final entry in data.entries) { + final key = entry.key; + var value = entry.value; + if (value is Map) value = deepCopyOut(value); + result[key] = value; + } + return result; + } + + @override + Uint8List serialize(Map data) => + (data as JsonBufferMap).serialize(); + + @override + Map deserialize(Uint8List data) => + JsonBuffer.deserialize(data).asMap; + + JsonBuffer _createLibrary({required int classCount}) { + return JsonBuffer( + keys: [for (var i = 0; i != classCount; ++i) 'A$i'], + function: (_) => _createClass(fieldCount: 10)); + } + + JsonBuffer _createClass({required int fieldCount}) { + return JsonBuffer( + keys: [for (var i = 0; i != fieldCount; ++i) 'f$i'], + function: (_) => _createField()); + } + + JsonBuffer _createField() { + return JsonBuffer( + keys: ['type', 'properties'], + function: (key) { + switch (key) { + case 'type': + return 'int'; + case 'properties': + return JsonBuffer( + keys: ['abstract', 'static', 'final'], + function: (key) { + switch (key) { + case 'abstract': + return true; + case 'final': + return false; + case 'static': + return false; + default: + throw StateError(key); + } + }); + default: + throw StateError(key); + } + }); + } +} diff --git a/working/macros/dart_model/testing/json_benchmark/lib/json_subject.dart b/working/macros/dart_model/testing/json_benchmark/lib/json_subject.dart new file mode 100644 index 0000000000..a573ab6b9f --- /dev/null +++ b/working/macros/dart_model/testing/json_benchmark/lib/json_subject.dart @@ -0,0 +1,69 @@ +import 'dart:convert'; +import 'dart:typed_data'; + +import 'subject.dart'; + +class JsonSubject implements Subject { + @override + String get name => 'JSON'; + + @override + Map createData({required int libraryCount}) { + final result = {}; + + for (var i = 0; i != libraryCount; ++i) { + final packageName = 'package:json_benchmark/library$i.dart'; + result[packageName] = _createLibrary(classCount: 10); + } + + return result; + } + + @override + Map deepCopyIn(Map data) { + final result = {}; + for (final entry in data.entries) { + final key = entry.key; + var value = entry.value; + if (value is Map) value = deepCopyIn(value); + result[key] = value; + } + return result; + } + + @override + Map deepCopyOut(Map data) => + deepCopyIn(data); + + @override + Uint8List serialize(Map data) => + utf8.encode(json.encode(data)); + @override + Map deserialize(List data) => + json.decode(utf8.decode(data)); + + Map _createLibrary({required int classCount}) { + final result = {}; + for (var i = 0; i != classCount; ++i) { + final className = 'A$i'; + result[className] = _createClass(fieldCount: 10); + } + return result; + } + + Map _createClass({required int fieldCount}) { + final result = {}; + for (var i = 0; i != fieldCount; ++i) { + final fieldName = 'f$i'; + result[fieldName] = _createField(); + } + return result; + } + + Map _createField() { + return { + 'type': 'int', + 'properties': {'abstract': true, 'final': true, 'static': false} + }; + } +} diff --git a/working/macros/dart_model/testing/json_benchmark/lib/subject.dart b/working/macros/dart_model/testing/json_benchmark/lib/subject.dart new file mode 100644 index 0000000000..775a7a2f3b --- /dev/null +++ b/working/macros/dart_model/testing/json_benchmark/lib/subject.dart @@ -0,0 +1,14 @@ +import 'dart:typed_data'; + +abstract class Subject { + String get name; + + Map createData({required int libraryCount}); + + Map deepCopyIn(Map data); + + Map deepCopyOut(Map data); + + Uint8List serialize(Map data); + Map deserialize(Uint8List data); +} diff --git a/working/macros/dart_model/testing/json_benchmark/pubspec.yaml b/working/macros/dart_model/testing/json_benchmark/pubspec.yaml new file mode 100644 index 0000000000..6067f9e5e9 --- /dev/null +++ b/working/macros/dart_model/testing/json_benchmark/pubspec.yaml @@ -0,0 +1,8 @@ +name: json_benchmark +publish-to: none + +environment: + sdk: ^3.4.0 + +dev_dependencies: + test: ^1.25.0 diff --git a/working/macros/dart_model/testing/json_benchmark/test/json_buffer_test.dart b/working/macros/dart_model/testing/json_benchmark/test/json_buffer_test.dart new file mode 100644 index 0000000000..431b6a2e39 --- /dev/null +++ b/working/macros/dart_model/testing/json_benchmark/test/json_buffer_test.dart @@ -0,0 +1,67 @@ +// Copyright (c) 2024, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'package:json_benchmark/json_buffer.dart'; +import 'package:test/test.dart'; + +void main() { + group(JsonBuffer, () { + test('map with string values', () { + final buffer = JsonBuffer( + keys: ['a', 'aa', 'bbb'], function: (key) => key.length.toString()); + + expect(buffer.asMap.keys, ['a', 'aa', 'bbb']); + expect(buffer.asMap, {'a': '1', 'aa': '2', 'bbb': '3'}); + }); + + test('map with bool values', () { + final buffer = + JsonBuffer(keys: ['a', 'aa', 'bbb'], function: (key) => key == 'aa'); + + expect(buffer.asMap.keys, ['a', 'aa', 'bbb']); + expect(buffer.asMap, {'a': false, 'aa': true, 'bbb': false}); + }); + + test('map with map values', () { + final buffer = JsonBuffer( + keys: ['a', 'aa', 'bbb'], + function: (key) => JsonBuffer( + keys: ['${key}1', '${key}2'], + function: (key) => key.substring(key.length - 1))); + + expect(buffer.asMap.keys, ['a', 'aa', 'bbb']); + expect(buffer.asMap, { + 'a': {'a1': '1', 'a2': '2'}, + 'aa': {'aa1': '1', 'aa2': '2'}, + 'bbb': {'bbb1': '1', 'bbb2': '2'}, + }); + }); + + test('serialization round trip', () { + final buffer = JsonBuffer( + keys: ['a', 'aa', 'bbb'], + function: (key) => JsonBuffer( + keys: ['${key}1', '${key}2'], + function: (key) => key.substring(key.length - 1))); + + final roundTripBuffer = JsonBuffer.deserialize(buffer.serialize()); + expect(roundTripBuffer.asMap, buffer.asMap); + }); + + test('serialization round trip large map', () { + final buffer = JsonBuffer( + keys: List.generate(1000000, (i) => i.toString()), + function: (key) => key.toString()); + + final roundTripBuffer = JsonBuffer.deserialize(buffer.serialize()); + expect(roundTripBuffer.asMap.keys.length, 1000000); + + // Don't use `expect` to compare for equality as it's quadratic in `Map` + // size. + expect(roundTripBuffer.asMap.keys.toList(), buffer.asMap.keys.toList()); + expect( + roundTripBuffer.asMap.values.toList(), buffer.asMap.values.toList()); + }); + }); +} diff --git a/working/macros/dart_model/testing/presubmit b/working/macros/dart_model/testing/presubmit index efb135beb9..5d9c2a346e 100755 --- a/working/macros/dart_model/testing/presubmit +++ b/working/macros/dart_model/testing/presubmit @@ -7,7 +7,8 @@ testing/pub_get # Analyze, format and test packages. for package in dart_model dart_model_analyzer_service dart_model_repl \ macro_client macro_host macro_protocol testing/scratch \ - testing/benchmark testing/test_macro_annotations testing/test_macros; do + testing/benchmark testing/json_benchmark testing/test_macro_annotations \ + testing/test_macros; do pushd "$package" dart analyze --fatal-infos diff --git a/working/macros/dart_model/testing/pub_get b/working/macros/dart_model/testing/pub_get index 9a8db9605d..a45f156c5f 100755 --- a/working/macros/dart_model/testing/pub_get +++ b/working/macros/dart_model/testing/pub_get @@ -4,7 +4,8 @@ set -e for package in dart_model dart_model_analyzer_service dart_model_repl \ macro_client macro_host macro_protocol testing/scratch \ - testing/benchmark testing/test_macro_annotations testing/test_macros; do + testing/benchmark testing/json_benchmark testing/test_macro_annotations \ + testing/test_macros; do pushd "$package" dart pub get popd