From 3fc704b5625283292c613f1c2ede4e9900e1b79a Mon Sep 17 00:00:00 2001 From: David Morgan Date: Mon, 10 Feb 2025 11:11:51 +0100 Subject: [PATCH] Expand benchmarks to different dep graph shapes. Also add: support for setting the benchmark size, support for ignoring failures, and support for the `--use-experimental-resolver` flag. --- _benchmark/bin/_benchmark.dart | 20 ++++ .../built_value_generator_benchmark.dart | 19 ++-- .../freezed_generator_benchmark.dart | 19 ++-- ...json_serializable_generator_benchmark.dart | 19 ++-- .../mockito_generator_benchmark.dart | 19 ++-- _benchmark/lib/commands.dart | 98 ++++++++++++------- _benchmark/lib/config.dart | 22 ++++- _benchmark/lib/shape.dart | 64 ++++++++++++ _benchmark/lib/workspace.dart | 19 ++-- 9 files changed, 216 insertions(+), 83 deletions(-) create mode 100644 _benchmark/lib/shape.dart diff --git a/_benchmark/bin/_benchmark.dart b/_benchmark/bin/_benchmark.dart index a078985bb..3dd611fc6 100644 --- a/_benchmark/bin/_benchmark.dart +++ b/_benchmark/bin/_benchmark.dart @@ -6,6 +6,7 @@ import 'dart:io'; import 'package:_benchmark/commands.dart'; import 'package:_benchmark/generators.dart'; +import 'package:_benchmark/shape.dart'; import 'package:args/command_runner.dart'; final commandRunner = @@ -16,6 +17,10 @@ final commandRunner = ..addCommand(BenchmarkCommand()) ..addCommand(MeasureCommand()) ..addCommand(CreateCommand()) + ..argParser.addFlag( + 'allow-failures', + help: 'Whether to continue benchmarking despite failures.', + ) ..argParser.addOption( 'build-repo-path', help: 'Path to build repo to benchmark.', @@ -30,6 +35,21 @@ final commandRunner = 'root-directory', help: 'Root directory for generated source and builds.', defaultsTo: '${Directory.systemTemp.path}/build_benchmark', + ) + ..argParser.addOption( + 'size', + help: + 'Benchmark size: number of libraries. Omit to run for a range of ' + 'sizes.', + ) + ..argParser.addOption( + 'shape', + help: 'Shape of the dependency graph. Omit to run for all shapes.', + allowed: Shape.values.map((e) => e.name).toList(), + ) + ..argParser.addFlag( + 'use-experimental-resolver', + help: 'Whether to pass `--use-experimental-resolver` to build_runner.', ); Future main(List arguments) async { diff --git a/_benchmark/lib/benchmarks/built_value_generator_benchmark.dart b/_benchmark/lib/benchmarks/built_value_generator_benchmark.dart index e137fe48a..04cb44250 100644 --- a/_benchmark/lib/benchmarks/built_value_generator_benchmark.dart +++ b/_benchmark/lib/benchmarks/built_value_generator_benchmark.dart @@ -38,17 +38,12 @@ ${config.dependencyOverrides} ''', ); - final appLines = ['// ignore_for_file: unused_import', '// CACHEBUSTER']; - for (var libraryNumber = 0; libraryNumber != size; ++libraryNumber) { - final libraryName = Benchmarks.libraryName( - libraryNumber, - benchmarkSize: size, - ); - appLines.add("import '$libraryName';"); - } workspace.write( 'lib/app.dart', - source: appLines.map((l) => '$l\n').join(''), + source: ''' +/// ignore_for_file: unused_import +/// CACHEBUSTER +''', ); for (var libraryNumber = 0; libraryNumber != size; ++libraryNumber) { @@ -57,13 +52,17 @@ ${config.dependencyOverrides} benchmarkSize: size, ); final partName = Benchmarks.partName(libraryNumber, benchmarkSize: size); + final importNames = config.shape.importNames( + libraryNumber, + benchmarkSize: size, + ); workspace.write( 'lib/$libraryName', source: ''' // ignore_for_file: unused_import import 'package:built_value/built_value.dart'; -import 'app.dart'; +${[for (final importName in importNames) "import '$importName';"].join('\n')} part '$partName'; diff --git a/_benchmark/lib/benchmarks/freezed_generator_benchmark.dart b/_benchmark/lib/benchmarks/freezed_generator_benchmark.dart index 5f5009a5a..8a2574324 100644 --- a/_benchmark/lib/benchmarks/freezed_generator_benchmark.dart +++ b/_benchmark/lib/benchmarks/freezed_generator_benchmark.dart @@ -38,17 +38,12 @@ ${config.dependencyOverrides} ''', ); - final appLines = ['// ignore_for_file: unused_import', '// CACHEBUSTER']; - for (var libraryNumber = 0; libraryNumber != size; ++libraryNumber) { - final libraryName = Benchmarks.libraryName( - libraryNumber, - benchmarkSize: size, - ); - appLines.add("import '$libraryName';"); - } workspace.write( 'lib/app.dart', - source: appLines.map((l) => '$l\n').join(''), + source: ''' +/// ignore_for_file: unused_import +/// CACHEBUSTER +''', ); for (var libraryNumber = 0; libraryNumber != size; ++libraryNumber) { @@ -61,13 +56,17 @@ ${config.dependencyOverrides} benchmarkSize: size, infix: 'freezed', ); + final importNames = config.shape.importNames( + libraryNumber, + benchmarkSize: size, + ); workspace.write( 'lib/$libraryName', source: ''' // ignore_for_file: unused_import import 'package:freezed_annotation/freezed_annotation.dart'; -import 'app.dart'; +${[for (final importName in importNames) "import '$importName';"].join('\n')} part '$partName'; diff --git a/_benchmark/lib/benchmarks/json_serializable_generator_benchmark.dart b/_benchmark/lib/benchmarks/json_serializable_generator_benchmark.dart index a965c1cb4..c8098357e 100644 --- a/_benchmark/lib/benchmarks/json_serializable_generator_benchmark.dart +++ b/_benchmark/lib/benchmarks/json_serializable_generator_benchmark.dart @@ -39,17 +39,12 @@ ${config.dependencyOverrides} ''', ); - final appLines = ['// ignore_for_file: unused_import', '// CACHEBUSTER']; - for (var libraryNumber = 0; libraryNumber != size; ++libraryNumber) { - final libraryName = Benchmarks.libraryName( - libraryNumber, - benchmarkSize: size, - ); - appLines.add("import '$libraryName';"); - } workspace.write( 'lib/app.dart', - source: appLines.map((l) => '$l\n').join(''), + source: ''' +/// ignore_for_file: unused_import +/// CACHEBUSTER +''', ); for (var libraryNumber = 0; libraryNumber != size; ++libraryNumber) { @@ -58,13 +53,17 @@ ${config.dependencyOverrides} benchmarkSize: size, ); final partName = Benchmarks.partName(libraryNumber, benchmarkSize: size); + final importNames = config.shape.importNames( + libraryNumber, + benchmarkSize: size, + ); workspace.write( 'lib/$libraryName', source: ''' // ignore_for_file: unused_import import 'package:json_annotation/json_annotation.dart'; -import 'app.dart'; +${[for (final importName in importNames) "import '$importName';"].join('\n')} part '$partName'; diff --git a/_benchmark/lib/benchmarks/mockito_generator_benchmark.dart b/_benchmark/lib/benchmarks/mockito_generator_benchmark.dart index 8882bb9a7..2eedff641 100644 --- a/_benchmark/lib/benchmarks/mockito_generator_benchmark.dart +++ b/_benchmark/lib/benchmarks/mockito_generator_benchmark.dart @@ -40,17 +40,12 @@ ${config.dependencyOverrides} ''', ); - final appLines = ['// ignore_for_file: unused_import', '// CACHEBUSTER']; - for (var libraryNumber = 0; libraryNumber != size; ++libraryNumber) { - final libraryName = Benchmarks.libraryName( - libraryNumber, - benchmarkSize: size, - ); - appLines.add("import '$libraryName';"); - } workspace.write( 'lib/app.dart', - source: appLines.map((l) => '$l\n').join(''), + source: ''' +/// ignore_for_file: unused_import +/// CACHEBUSTER +''', ); for (var testNumber = 0; testNumber != size; ++testNumber) { @@ -81,11 +76,15 @@ ${config.dependencyOverrides} libraryNumber, benchmarkSize: size, ); + final importNames = config.shape.importNames( + libraryNumber, + benchmarkSize: size, + ); workspace.write( 'lib/$libraryName', source: ''' // ignore_for_file: unused_import -import 'app.dart'; +${[for (final importName in importNames) "import '$importName';"].join('\n')} class Service$libraryNumber { void doSomething(int value) {} diff --git a/_benchmark/lib/commands.dart b/_benchmark/lib/commands.dart index c0f9cc221..350e1d07a 100644 --- a/_benchmark/lib/commands.dart +++ b/_benchmark/lib/commands.dart @@ -6,6 +6,7 @@ import 'package:args/args.dart'; import 'package:args/command_runner.dart'; import 'config.dart'; +import 'shape.dart'; import 'workspace.dart'; class BenchmarkCommand extends Command { @@ -35,19 +36,22 @@ class CreateCommand extends Command { Future _run(ArgResults globalResults) async { final config = Config.fromArgResults(globalResults); - for (final size in config.sizes) { - final paddedSize = size.toString().padLeft(4, '0'); - final workspace = Workspace( - config: config, - name: '${config.generator.packageName}_$paddedSize', - ); - final runConfig = RunConfig( - config: config, - workspace: workspace, - size: size, - paddedSize: paddedSize, - ); - config.generator.benchmark.create(runConfig); + for (final shape in config.shapes) { + for (final size in config.sizes) { + final paddedSize = size.toString().padLeft(4, '0'); + final workspace = Workspace( + config: config, + name: '${config.generator.packageName}_${shape.name}_$paddedSize', + ); + final runConfig = RunConfig( + config: config, + workspace: workspace, + size: size, + paddedSize: paddedSize, + shape: shape, + ); + config.generator.benchmark.create(runConfig); + } } } } @@ -65,15 +69,17 @@ class MeasureCommand extends Command { Future _run(ArgResults globalResults) async { // Launch a benchmark at each size in parallel. final config = Config.fromArgResults(globalResults); - final pendingResults = {}; - for (final size in config.sizes) { - final paddedSize = size.toString().padLeft(4, '0'); - final workspace = Workspace( - config: config, - name: '${config.generator.packageName}_$paddedSize', - clean: false, - ); - pendingResults[size] = workspace.measure(); + final pendingResults = <(Shape, int), PendingResult>{}; + for (final shape in config.shapes) { + for (final size in config.sizes) { + final paddedSize = size.toString().padLeft(4, '0'); + final workspace = Workspace( + config: config, + name: '${config.generator.packageName}_${shape.name}_$paddedSize', + clean: false, + ); + pendingResults[(shape, size)] = workspace.measure(); + } } // Wait for them to complete, printing status every second only if it @@ -83,21 +89,36 @@ class MeasureCommand extends Command { await Future.delayed(const Duration(seconds: 1)); final update = StringBuffer('${config.generator.packageName}\n'); - update.write('libraries,clean/ms,no changes/ms,incremental/ms\n'); - for (final size in config.sizes) { - final pendingResult = pendingResults[size]!; - if (pendingResult.isFailure) { - throw StateError(pendingResult.failure!); + update.write('shape,libraries,clean/ms,no changes/ms,incremental/ms\n'); + for (final shape in config.shapes) { + for (final size in config.sizes) { + final pendingResult = pendingResults[(shape, size)]!; + if (pendingResult.isFailure) { + if (!config.allowFailures) { + throw StateError(pendingResult.failure!); + } + update.write( + [ + shape.name, + size, + pendingResult.cleanBuildTime.renderFailed, + pendingResult.noChangesBuildTime.renderFailed, + pendingResult.incrementalBuildTime.renderFailed, + ].join(','), + ); + } else { + update.write( + [ + shape.name, + size, + pendingResult.cleanBuildTime.render, + pendingResult.noChangesBuildTime.render, + pendingResult.incrementalBuildTime.render, + ].join(','), + ); + } + update.write('\n'); } - update.write( - [ - size, - pendingResults[size]!.cleanBuildTime.render, - pendingResults[size]!.noChangesBuildTime.render, - pendingResults[size]!.incrementalBuildTime.render, - ].join(','), - ); - update.write('\n'); } final updateString = update.toString(); @@ -110,5 +131,10 @@ class MeasureCommand extends Command { } extension DurationExtension on Duration? { + /// Renders with `---` for `null`, to mean "pending". String get render => this == null ? '---' : this!.inMilliseconds.toString(); + + /// Renders with X` for `null`, to mean "failed". + String get renderFailed => + this == null ? 'X' : this!.inMilliseconds.toString(); } diff --git a/_benchmark/lib/config.dart b/_benchmark/lib/config.dart index 9476d4c4b..f5c4d7aeb 100644 --- a/_benchmark/lib/config.dart +++ b/_benchmark/lib/config.dart @@ -7,27 +7,45 @@ import 'dart:io'; import 'package:args/args.dart'; import 'generators.dart'; +import 'shape.dart'; import 'workspace.dart'; /// Benchmark tool config. class Config { final String? buildRepoPath; final Generator generator; + final List shapes; final Directory rootDirectory; - final List sizes = const [1, 100, 250, 500, 750, 1000]; + final List sizes; + final bool useExperimentalResolver; + final bool allowFailures; Config({ + required this.allowFailures, required this.buildRepoPath, required this.generator, required this.rootDirectory, + required this.sizes, + required this.shapes, + required this.useExperimentalResolver, }); factory Config.fromArgResults(ArgResults argResults) => Config( + allowFailures: argResults['allow-failures'] as bool, buildRepoPath: argResults['build-repo-path'] as String?, generator: Generator.values.singleWhere( (e) => e.packageName == argResults['generator'], ), rootDirectory: Directory(argResults['root-directory'] as String), + sizes: + argResults['size'] == null + ? [1, 100, 250, 500, 750, 1000] + : [int.parse(argResults['size'] as String)], + shapes: + argResults['shape'] == null + ? Shape.values + : [Shape.values.singleWhere((e) => e.name == argResults['shape'])], + useExperimentalResolver: argResults['use-experimental-resolver'] as bool, ); } @@ -35,6 +53,7 @@ class Config { class RunConfig { final Config config; final int size; + final Shape shape; /// [size] as a padded-to-consistent-width `String`. final String paddedSize; @@ -46,6 +65,7 @@ class RunConfig { required this.workspace, required this.size, required this.paddedSize, + required this.shape, }); String get dependencyOverrides { diff --git a/_benchmark/lib/shape.dart b/_benchmark/lib/shape.dart new file mode 100644 index 000000000..621c5fbac --- /dev/null +++ b/_benchmark/lib/shape.dart @@ -0,0 +1,64 @@ +// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'benchmark.dart'; + +/// The shape of the dependency graph. +/// +/// Ordering matters because dependencies are read as needed for each build. If +/// the first build already depends on everything, as in `forwards`, that's very +/// different to if each build brings in a new dependency, as in `backwards`. +/// +/// `app.dart` is the file that will be changed to trigger an incremental build, +/// in all cases all files transitively depend on it. +enum Shape { + /// A single library cycle. + loop, + + /// A line of libraries. + /// + /// Considered in alphanumeric order, each one has a dependency on the next. + /// The final library depends on `app.dart`. + forwards, + + /// A line of libraries. + /// + /// Considered in alphanumeric order, each one has a dependency on the one + /// before. The first library depends on `app.dart`. + backwards; + + Iterable importNames( + int libraryNumber, { + required int benchmarkSize, + }) { + switch (this) { + case Shape.loop: + return [ + if (libraryNumber == 0) 'app.dart', + Benchmarks.libraryName( + (libraryNumber - 1) % benchmarkSize, + benchmarkSize: benchmarkSize, + ), + ]; + case Shape.forwards: + return [ + if (libraryNumber == benchmarkSize - 1) 'app.dart', + if (libraryNumber != benchmarkSize - 1) + Benchmarks.libraryName( + libraryNumber + 1, + benchmarkSize: benchmarkSize, + ), + ]; + case Shape.backwards: + return [ + if (libraryNumber == 0) 'app.dart', + if (libraryNumber != 0) + Benchmarks.libraryName( + libraryNumber - 1, + benchmarkSize: benchmarkSize, + ), + ]; + } + } +} diff --git a/_benchmark/lib/workspace.dart b/_benchmark/lib/workspace.dart index beea87a85..fb47ddcee 100644 --- a/_benchmark/lib/workspace.dart +++ b/_benchmark/lib/workspace.dart @@ -60,10 +60,12 @@ class Workspace { 'build_runner', 'build', '-d', + if (config.useExperimentalResolver) '--use-experimental-resolver', ], workingDirectory: directory.path); var exitCode = await process.exitCode; - result.cleanBuildTime = stopwatch.elapsed; - if (exitCode != 0) { + if (exitCode == 0) { + result.cleanBuildTime = stopwatch.elapsed; + } else { final stdout = await process.stdout.transform(utf8.decoder).join(); final stderr = await process.stderr.transform(utf8.decoder).join(); result.failure = 'Initial build failed:\n$stdout\n$stderr'; @@ -77,10 +79,12 @@ class Workspace { 'build_runner', 'build', '-d', + if (config.useExperimentalResolver) '--use-experimental-resolver', ], workingDirectory: directory.path); exitCode = await process.exitCode; - result.noChangesBuildTime = stopwatch.elapsed; - if (exitCode != 0) { + if (exitCode == 0) { + result.noChangesBuildTime = stopwatch.elapsed; + } else { final stdout = await process.stdout.transform(utf8.decoder).join(); final stderr = await process.stderr.transform(utf8.decoder).join(); result.failure = 'No changes build failed:\n$stdout\n$stderr'; @@ -95,10 +99,12 @@ class Workspace { 'build_runner', 'build', '-d', + if (config.useExperimentalResolver) '--use-experimental-resolver', ], workingDirectory: directory.path); exitCode = await process.exitCode; - result.incrementalBuildTime = stopwatch.elapsed; - if (exitCode != 0) { + if (exitCode == 0) { + result.incrementalBuildTime = stopwatch.elapsed; + } else { final stdout = await process.stdout.transform(utf8.decoder).join(); final stderr = await process.stderr.transform(utf8.decoder).join(); result.failure = 'Incremental build failed:\n$stdout\n$stderr'; @@ -117,6 +123,7 @@ class PendingResult { String? failure; bool get isFailure => failure != null; + bool get isSuccess => !isFailure && cleanBuildTime != null &&