Skip to content

Commit

Permalink
Switch deps crawl to iterative to prevent stack overflows.
Browse files Browse the repository at this point in the history
  • Loading branch information
davidmorgan committed Feb 10, 2025
1 parent 3fc704b commit c1e2fe6
Show file tree
Hide file tree
Showing 5 changed files with 1,032 additions and 566 deletions.
2 changes: 2 additions & 0 deletions build_resolvers/CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@
- Make resolver only throw `SyntaxErrorInAssetException` on severe syntax errors
- Add `BuildAssetUriResolver.useExperimentalResolver` for
`--use-experimental-resolver` flag. This will be removed.
- Switch `BuildAssetUriResolver` dependency crawl to an iterative
algorithm, preventing stack overflows.

## 2.4.3

Expand Down
2 changes: 1 addition & 1 deletion build_resolvers/lib/src/build_asset_uri_resolver.dart
Original file line number Diff line number Diff line change
Expand Up @@ -13,11 +13,11 @@ import 'package:analyzer/src/clients/build_resolvers/build_resolvers.dart';
import 'package:build/build.dart' show AssetId, BuildStep;
import 'package:collection/collection.dart';
import 'package:crypto/crypto.dart';
import 'package:graphs/graphs.dart';
import 'package:path/path.dart' as p;

import 'analysis_driver_filesystem.dart';
import 'analysis_driver_model.dart';
import 'crawl_async.dart';

const _ignoredSchemes = ['dart', 'dart-ext'];

Expand Down
83 changes: 83 additions & 0 deletions build_resolvers/lib/src/crawl_async.dart
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.

import 'dart:async';
import 'dart:collection';

final _empty = Future<void>.value();

/// Finds and returns every node in a graph who's nodes and edges are
/// asynchronously resolved.
///
/// Cycles are allowed. If this is an undirected graph the [edges] function
/// may be symmetric. In this case the [roots] may be any node in each connected
/// graph.
///
/// [V] is the type of values in the graph nodes. [K] must be a type suitable
/// for using as a Map or Set key. [edges] should return the next reachable
/// nodes.
///
/// There are no ordering guarantees. This is useful for ensuring some work is
/// performed at every node in an asynchronous graph, but does not give
/// guarantees that the work is done in topological order.
///
/// If either [readNode] or [edges] throws the error will be forwarded
/// through the result stream and no further nodes will be crawled, though some
/// work may have already been started.
///
/// Crawling is eager, so calls to [edges] may overlap with other calls that
/// have not completed. If the [edges] callback needs to be limited or throttled
/// that must be done by wrapping it before calling [crawlAsync].
///
/// This is a fork of the `package:graph` algorithm changed from recursive to
/// iterative; it is mostly for benchmarking, as `AnalysisDriverModel` will
/// replace the use of this method entirely.
Stream<V> crawlAsync<K extends Object, V>(
Iterable<K> roots,
FutureOr<V> Function(K) readNode,
FutureOr<Iterable<K>> Function(K, V) edges,
) {
final crawl = _CrawlAsync(roots, readNode, edges)..run();
return crawl.result.stream;
}

class _CrawlAsync<K, V> {
final result = StreamController<V>();

final FutureOr<V> Function(K) readNode;
final FutureOr<Iterable<K>> Function(K, V) edges;
final Iterable<K> roots;

final _seen = HashSet<K>();
final _next = Queue<K>();

_CrawlAsync(this.roots, this.readNode, this.edges);

/// Add all nodes in the graph to [result] and return a Future which fires
/// after all nodes have been seen.
Future<void> run() async {
try {
_next.addAll(roots);
while (_next.isNotEmpty) {
await _crawlNext();
}
await result.close();
} catch (e, st) {
result.addError(e, st);
await result.close();
}
}

/// Remove the next `key` from [_next], queue up any of its its edges
/// that haven't been seen.
Future<void> _crawlNext() async {
final key = _next.removeFirst();
final value = await readNode(key);
if (result.isClosed) return;
result.add(value);
for (final edge in await edges(key, value)) {
if (_seen.add(edge)) _next.add(edge);
}
}
}
130 changes: 130 additions & 0 deletions build_resolvers/test/crawl_async_test.dart
Original file line number Diff line number Diff line change
@@ -0,0 +1,130 @@
// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.

import 'package:build_resolvers/src/crawl_async.dart';
import 'package:graphs/graphs.dart' hide crawlAsync;
import 'package:test/test.dart';

// This is a fork of the `package:graph` test of the same name, there are
// no changes. TODO(davidmorgan): remove when `crawlAsync` is removed
// in favor of the new `AnalysisDriverModel`.

void main() {
group('asyncCrawl', () {
Future<List<String?>> crawl(
Map<String, List<String>?> g,
Iterable<String> roots,
) {
final graph = AsyncGraph(g);
return crawlAsync(roots, graph.readNode, graph.edges).toList();
}

test('empty result for empty graph', () async {
final result = await crawl({}, []);
expect(result, isEmpty);
});

test('single item for a single node', () async {
final result = await crawl({'a': []}, ['a']);
expect(result, ['a']);
});

test('hits every node in a graph', () async {
final result = await crawl({
'a': ['b', 'c'],
'b': ['c'],
'c': ['d'],
'd': [],
}, [
'a',
]);
expect(result, hasLength(4));
expect(
result,
allOf(contains('a'), contains('b'), contains('c'), contains('d')),
);
});

test('handles cycles', () async {
final result = await crawl({
'a': ['b'],
'b': ['c'],
'c': ['b'],
}, [
'a',
]);
expect(result, hasLength(3));
expect(result, allOf(contains('a'), contains('b'), contains('c')));
});

test('handles self cycles', () async {
final result = await crawl({
'a': ['b'],
'b': ['b'],
}, [
'a',
]);
expect(result, hasLength(2));
expect(result, allOf(contains('a'), contains('b')));
});

test('allows null edges', () async {
final result = await crawl({
'a': ['b'],
'b': null,
}, [
'a',
]);
expect(result, hasLength(2));
expect(result, allOf(contains('a'), contains('b')));
});

test('allows null nodes', () async {
final result = await crawl({
'a': ['b'],
}, [
'a',
]);
expect(result, ['a', null]);
});

test('surfaces exceptions for crawling edges', () {
final graph = {
'a': ['b'],
};
final nodes = crawlAsync(
['a'],
(n) => n,
(k, n) => k == 'b' ? throw ArgumentError() : graph[k] ?? <String>[],
);
expect(nodes, emitsThrough(emitsError(isArgumentError)));
});

test('surfaces exceptions for resolving keys', () {
final graph = {
'a': ['b'],
};
final nodes = crawlAsync(
['a'],
(n) => n == 'b' ? throw ArgumentError() : n,
(k, n) => graph[k] ?? <Never>[],
);
expect(nodes, emitsThrough(emitsError(isArgumentError)));
});
});
}

/// A representation of a Graph where keys can asynchronously be resolved to
/// real values or to edges.
class AsyncGraph {
final Map<String, List<String>?> graph;

AsyncGraph(this.graph);

Future<String?> readNode(String node) async =>
graph.containsKey(node) ? node : null;

Future<Iterable<String>> edges(String key, String? node) async =>
graph[key] ?? <Never>[];
}
Loading

0 comments on commit c1e2fe6

Please sign in to comment.