Expand benchmarks to different dep graph shapes.

Also add: support for setting the benchmark size, support for ignoring failures, and support for the `--use-experimental-resolver` flag.
dart-lang · Feb 10, 2025 · 3fc704b · 3fc704b
1 parent 25d5f61
commit 3fc704b
Show file tree

Hide file tree

Showing 9 changed files with 216 additions and 83 deletions.
diff --git a/_benchmark/bin/_benchmark.dart b/_benchmark/bin/_benchmark.dart
@@ -6,6 +6,7 @@ import 'dart:io';
 
 import 'package:_benchmark/commands.dart';
 import 'package:_benchmark/generators.dart';
+import 'package:_benchmark/shape.dart';
 import 'package:args/command_runner.dart';
 
 final commandRunner =
@@ -16,6 +17,10 @@ final commandRunner =
       ..addCommand(BenchmarkCommand())
       ..addCommand(MeasureCommand())
       ..addCommand(CreateCommand())
+      ..argParser.addFlag(
+        'allow-failures',
+        help: 'Whether to continue benchmarking despite failures.',
+      )
       ..argParser.addOption(
         'build-repo-path',
         help: 'Path to build repo to benchmark.',
@@ -30,6 +35,21 @@ final commandRunner =
         'root-directory',
         help: 'Root directory for generated source and builds.',
         defaultsTo: '${Directory.systemTemp.path}/build_benchmark',
+      )
+      ..argParser.addOption(
+        'size',
+        help:
+            'Benchmark size: number of libraries. Omit to run for a range of '
+            'sizes.',
+      )
+      ..argParser.addOption(
+        'shape',
+        help: 'Shape of the dependency graph. Omit to run for all shapes.',
+        allowed: Shape.values.map((e) => e.name).toList(),
+      )
+      ..argParser.addFlag(
+        'use-experimental-resolver',
+        help: 'Whether to pass `--use-experimental-resolver` to build_runner.',
       );
 
 Future<void> main(List<String> arguments) async {

diff --git a/_benchmark/lib/benchmarks/built_value_generator_benchmark.dart b/_benchmark/lib/benchmarks/built_value_generator_benchmark.dart
@@ -38,17 +38,12 @@ ${config.dependencyOverrides}
 ''',
     );
 
-    final appLines = ['// ignore_for_file: unused_import', '// CACHEBUSTER'];
-    for (var libraryNumber = 0; libraryNumber != size; ++libraryNumber) {
-      final libraryName = Benchmarks.libraryName(
-        libraryNumber,
-        benchmarkSize: size,
-      );
-      appLines.add("import '$libraryName';");
-    }
     workspace.write(
       'lib/app.dart',
-      source: appLines.map((l) => '$l\n').join(''),
+      source: '''
+/// ignore_for_file: unused_import
+/// CACHEBUSTER
+''',
     );
 
     for (var libraryNumber = 0; libraryNumber != size; ++libraryNumber) {
@@ -57,13 +52,17 @@ ${config.dependencyOverrides}
         benchmarkSize: size,
       );
       final partName = Benchmarks.partName(libraryNumber, benchmarkSize: size);
+      final importNames = config.shape.importNames(
+        libraryNumber,
+        benchmarkSize: size,
+      );
       workspace.write(
         'lib/$libraryName',
         source: '''
 // ignore_for_file: unused_import
 import 'package:built_value/built_value.dart';
 
-import 'app.dart';
+${[for (final importName in importNames) "import '$importName';"].join('\n')}
 
 part '$partName';
 

diff --git a/_benchmark/lib/benchmarks/freezed_generator_benchmark.dart b/_benchmark/lib/benchmarks/freezed_generator_benchmark.dart
@@ -38,17 +38,12 @@ ${config.dependencyOverrides}
 ''',
     );
 
-    final appLines = ['// ignore_for_file: unused_import', '// CACHEBUSTER'];
-    for (var libraryNumber = 0; libraryNumber != size; ++libraryNumber) {
-      final libraryName = Benchmarks.libraryName(
-        libraryNumber,
-        benchmarkSize: size,
-      );
-      appLines.add("import '$libraryName';");
-    }
     workspace.write(
       'lib/app.dart',
-      source: appLines.map((l) => '$l\n').join(''),
+      source: '''
+/// ignore_for_file: unused_import
+/// CACHEBUSTER
+''',
     );
 
     for (var libraryNumber = 0; libraryNumber != size; ++libraryNumber) {
@@ -61,13 +56,17 @@ ${config.dependencyOverrides}
         benchmarkSize: size,
         infix: 'freezed',
       );
+      final importNames = config.shape.importNames(
+        libraryNumber,
+        benchmarkSize: size,
+      );
       workspace.write(
         'lib/$libraryName',
         source: '''
 // ignore_for_file: unused_import
 import 'package:freezed_annotation/freezed_annotation.dart';
 
-import 'app.dart';
+${[for (final importName in importNames) "import '$importName';"].join('\n')}
 
 part '$partName';
 

diff --git a/_benchmark/lib/benchmarks/json_serializable_generator_benchmark.dart b/_benchmark/lib/benchmarks/json_serializable_generator_benchmark.dart
@@ -39,17 +39,12 @@ ${config.dependencyOverrides}
 ''',
     );
 
-    final appLines = ['// ignore_for_file: unused_import', '// CACHEBUSTER'];
-    for (var libraryNumber = 0; libraryNumber != size; ++libraryNumber) {
-      final libraryName = Benchmarks.libraryName(
-        libraryNumber,
-        benchmarkSize: size,
-      );
-      appLines.add("import '$libraryName';");
-    }
     workspace.write(
       'lib/app.dart',
-      source: appLines.map((l) => '$l\n').join(''),
+      source: '''
+/// ignore_for_file: unused_import
+/// CACHEBUSTER
+''',
     );
 
     for (var libraryNumber = 0; libraryNumber != size; ++libraryNumber) {
@@ -58,13 +53,17 @@ ${config.dependencyOverrides}
         benchmarkSize: size,
       );
       final partName = Benchmarks.partName(libraryNumber, benchmarkSize: size);
+      final importNames = config.shape.importNames(
+        libraryNumber,
+        benchmarkSize: size,
+      );
       workspace.write(
         'lib/$libraryName',
         source: '''
 // ignore_for_file: unused_import
 import 'package:json_annotation/json_annotation.dart';
 
-import 'app.dart';
+${[for (final importName in importNames) "import '$importName';"].join('\n')}
 
 part '$partName';
 

diff --git a/_benchmark/lib/benchmarks/mockito_generator_benchmark.dart b/_benchmark/lib/benchmarks/mockito_generator_benchmark.dart
@@ -40,17 +40,12 @@ ${config.dependencyOverrides}
 ''',
     );
 
-    final appLines = ['// ignore_for_file: unused_import', '// CACHEBUSTER'];
-    for (var libraryNumber = 0; libraryNumber != size; ++libraryNumber) {
-      final libraryName = Benchmarks.libraryName(
-        libraryNumber,
-        benchmarkSize: size,
-      );
-      appLines.add("import '$libraryName';");
-    }
     workspace.write(
       'lib/app.dart',
-      source: appLines.map((l) => '$l\n').join(''),
+      source: '''
+/// ignore_for_file: unused_import
+/// CACHEBUSTER
+''',
     );
 
     for (var testNumber = 0; testNumber != size; ++testNumber) {
@@ -81,11 +76,15 @@ ${config.dependencyOverrides}
         libraryNumber,
         benchmarkSize: size,
       );
+      final importNames = config.shape.importNames(
+        libraryNumber,
+        benchmarkSize: size,
+      );
       workspace.write(
         'lib/$libraryName',
         source: '''
 // ignore_for_file: unused_import
-import 'app.dart';
+${[for (final importName in importNames) "import '$importName';"].join('\n')}
 
 class Service$libraryNumber {
   void doSomething(int value) {}

diff --git a/_benchmark/lib/commands.dart b/_benchmark/lib/commands.dart
@@ -6,6 +6,7 @@ import 'package:args/args.dart';
 import 'package:args/command_runner.dart';
 
 import 'config.dart';
+import 'shape.dart';
 import 'workspace.dart';
 
 class BenchmarkCommand extends Command<void> {
@@ -35,19 +36,22 @@ class CreateCommand extends Command<void> {
   Future<void> _run(ArgResults globalResults) async {
     final config = Config.fromArgResults(globalResults);
 
-    for (final size in config.sizes) {
-      final paddedSize = size.toString().padLeft(4, '0');
-      final workspace = Workspace(
-        config: config,
-        name: '${config.generator.packageName}_$paddedSize',
-      );
-      final runConfig = RunConfig(
-        config: config,
-        workspace: workspace,
-        size: size,
-        paddedSize: paddedSize,
-      );
-      config.generator.benchmark.create(runConfig);
+    for (final shape in config.shapes) {
+      for (final size in config.sizes) {
+        final paddedSize = size.toString().padLeft(4, '0');
+        final workspace = Workspace(
+          config: config,
+          name: '${config.generator.packageName}_${shape.name}_$paddedSize',
+        );
+        final runConfig = RunConfig(
+          config: config,
+          workspace: workspace,
+          size: size,
+          paddedSize: paddedSize,
+          shape: shape,
+        );
+        config.generator.benchmark.create(runConfig);
+      }
     }
   }
 }
@@ -65,15 +69,17 @@ class MeasureCommand extends Command<void> {
   Future<void> _run(ArgResults globalResults) async {
     // Launch a benchmark at each size in parallel.
     final config = Config.fromArgResults(globalResults);
-    final pendingResults = <int, PendingResult>{};
-    for (final size in config.sizes) {
-      final paddedSize = size.toString().padLeft(4, '0');
-      final workspace = Workspace(
-        config: config,
-        name: '${config.generator.packageName}_$paddedSize',
-        clean: false,
-      );
-      pendingResults[size] = workspace.measure();
+    final pendingResults = <(Shape, int), PendingResult>{};
+    for (final shape in config.shapes) {
+      for (final size in config.sizes) {
+        final paddedSize = size.toString().padLeft(4, '0');
+        final workspace = Workspace(
+          config: config,
+          name: '${config.generator.packageName}_${shape.name}_$paddedSize',
+          clean: false,
+        );
+        pendingResults[(shape, size)] = workspace.measure();
+      }
     }
 
     // Wait for them to complete, printing status every second only if it
@@ -83,21 +89,36 @@ class MeasureCommand extends Command<void> {
       await Future<void>.delayed(const Duration(seconds: 1));
 
       final update = StringBuffer('${config.generator.packageName}\n');
-      update.write('libraries,clean/ms,no changes/ms,incremental/ms\n');
-      for (final size in config.sizes) {
-        final pendingResult = pendingResults[size]!;
-        if (pendingResult.isFailure) {
-          throw StateError(pendingResult.failure!);
+      update.write('shape,libraries,clean/ms,no changes/ms,incremental/ms\n');
+      for (final shape in config.shapes) {
+        for (final size in config.sizes) {
+          final pendingResult = pendingResults[(shape, size)]!;
+          if (pendingResult.isFailure) {
+            if (!config.allowFailures) {
+              throw StateError(pendingResult.failure!);
+            }
+            update.write(
+              [
+                shape.name,
+                size,
+                pendingResult.cleanBuildTime.renderFailed,
+                pendingResult.noChangesBuildTime.renderFailed,
+                pendingResult.incrementalBuildTime.renderFailed,
+              ].join(','),
+            );
+          } else {
+            update.write(
+              [
+                shape.name,
+                size,
+                pendingResult.cleanBuildTime.render,
+                pendingResult.noChangesBuildTime.render,
+                pendingResult.incrementalBuildTime.render,
+              ].join(','),
+            );
+          }
+          update.write('\n');
         }
-        update.write(
-          [
-            size,
-            pendingResults[size]!.cleanBuildTime.render,
-            pendingResults[size]!.noChangesBuildTime.render,
-            pendingResults[size]!.incrementalBuildTime.render,
-          ].join(','),
-        );
-        update.write('\n');
       }
 
       final updateString = update.toString();
@@ -110,5 +131,10 @@ class MeasureCommand extends Command<void> {
 }
 
 extension DurationExtension on Duration? {
+  /// Renders with `---` for `null`, to mean "pending".
   String get render => this == null ? '---' : this!.inMilliseconds.toString();
+
+  /// Renders with X` for `null`, to mean "failed".
+  String get renderFailed =>
+      this == null ? 'X' : this!.inMilliseconds.toString();
 }
diff --git a/_benchmark/lib/config.dart b/_benchmark/lib/config.dart
@@ -7,34 +7,53 @@ import 'dart:io';
 import 'package:args/args.dart';
 
 import 'generators.dart';
+import 'shape.dart';
 import 'workspace.dart';
 
 /// Benchmark tool config.
 class Config {
   final String? buildRepoPath;
   final Generator generator;
+  final List<Shape> shapes;
   final Directory rootDirectory;
-  final List<int> sizes = const [1, 100, 250, 500, 750, 1000];
+  final List<int> sizes;
+  final bool useExperimentalResolver;
+  final bool allowFailures;
 
   Config({
+    required this.allowFailures,
     required this.buildRepoPath,
     required this.generator,
     required this.rootDirectory,
+    required this.sizes,
+    required this.shapes,
+    required this.useExperimentalResolver,
   });
 
   factory Config.fromArgResults(ArgResults argResults) => Config(
+    allowFailures: argResults['allow-failures'] as bool,
     buildRepoPath: argResults['build-repo-path'] as String?,
     generator: Generator.values.singleWhere(
       (e) => e.packageName == argResults['generator'],
     ),
     rootDirectory: Directory(argResults['root-directory'] as String),
+    sizes:
+        argResults['size'] == null
+            ? [1, 100, 250, 500, 750, 1000]
+            : [int.parse(argResults['size'] as String)],
+    shapes:
+        argResults['shape'] == null
+            ? Shape.values
+            : [Shape.values.singleWhere((e) => e.name == argResults['shape'])],
+    useExperimentalResolver: argResults['use-experimental-resolver'] as bool,
   );
 }
 
 /// Single benchmark run config.
 class RunConfig {
   final Config config;
   final int size;
+  final Shape shape;
 
   /// [size] as a padded-to-consistent-width `String`.
   final String paddedSize;
@@ -46,6 +65,7 @@ class RunConfig {
     required this.workspace,
     required this.size,
     required this.paddedSize,
+    required this.shape,
   });
 
   String get dependencyOverrides {