extract engine sub-metrics; change reported metrics (flutter#55331)

yjbanov · web-flow · commit 6f8945fce65a · 2020-04-22T17:18:53.000-07:00
* extract engine sub-metrics; change reported metrics

- Extract sub-metrics reported by the Web engine: "preroll_frame", "apply_frame".
- Add a concept of unreported metrics: displayed on the benchmark UI, but not on the dashboard.
- Make "sceneBuildDuration" and "windowRenderDuration" unreported, which are too fine-grained. They are included in "drawFrameDuration" already.
- Report outlier ratio instead of outlier average. The ratio is more useful of the two.
diff --git a/dev/benchmarks/macrobenchmarks/lib/src/web/bench_picture_recording.dart b/dev/benchmarks/macrobenchmarks/lib/src/web/bench_picture_recording.dart
@@ -67,9 +67,9 @@ class BenchPictureRecording extends RawRecorder {
         }
         canvas.restore();
       }
-    });
+    }, reported: true);
     profile.record('estimatePaintBounds', () {
       recorder.endRecording();
-    });
+    }, reported: true);
   }
 }
diff --git a/dev/benchmarks/macrobenchmarks/lib/src/web/bench_text_layout.dart b/dev/benchmarks/macrobenchmarks/lib/src/web/bench_text_layout.dart
@@ -58,11 +58,6 @@ void _useCanvasText(bool useCanvasText) {
   );
 }
 
-typedef OnBenchmark = void Function(String name, num value);
-void _onBenchmark(OnBenchmark listener) {
-  js_util.setProperty(html.window, '_flutter_internal_on_benchmark', listener);
-}
-
 /// Repeatedly lays out a paragraph using the DOM measurement approach.
 ///
 /// Creates a different paragraph each time in order to avoid hitting the cache.
@@ -132,21 +127,21 @@ class BenchTextLayout extends RawRecorder {
   }) {
     profile.record('$keyPrefix.layout', () {
       paragraph.layout(ui.ParagraphConstraints(width: maxWidth));
-    });
+    }, reported: true);
     profile.record('$keyPrefix.getBoxesForRange', () {
       for (int start = 0; start < text.length; start += 3) {
         for (int end = start + 1; end < text.length; end *= 2) {
           paragraph.getBoxesForRange(start, end);
         }
       }
-    });
+    }, reported: true);
     profile.record('$keyPrefix.getPositionForOffset', () {
       for (double dx = 0.0; dx < paragraph.width; dx += 10.0) {
         for (double dy = 0.0; dy < paragraph.height; dy += 10.0) {
           paragraph.getPositionForOffset(Offset(dx, dy));
         }
       }
-    });
+    }, reported: true);
   }
 }
 
@@ -179,7 +174,7 @@ class BenchTextCachedLayout extends RawRecorder {
     final ui.Paragraph paragraph = builder.build();
     profile.record('layout', () {
       paragraph.layout(const ui.ParagraphConstraints(width: double.infinity));
-    });
+    }, reported: true);
     _useCanvasText(null);
   }
 }
@@ -242,15 +237,15 @@ class BenchBuildColorsGrid extends WidgetBuildRecorder {
     if (mode == _TestMode.useDomTextLayout) {
       _useCanvasText(false);
     }
-    _onBenchmark((String name, num value) {
+    registerEngineBenchmarkValueListener('text_layout', (num value) {
       _textLayoutMicros += value;
     });
   }
 
   @override
   Future<void> tearDownAll() async {
     _useCanvasText(null);
-    _onBenchmark(null);
+    stopListeningToEngineBenchmarkValues('text_layout');
   }
 
   @override
@@ -268,6 +263,7 @@ class BenchBuildColorsGrid extends WidgetBuildRecorder {
       profile.addDataPoint(
         'text_layout',
         Duration(microseconds: _textLayoutMicros.toInt()),
+        reported: true,
       );
     }
     super.frameDidDraw();
diff --git a/dev/benchmarks/macrobenchmarks/lib/src/web/recorder.dart b/dev/benchmarks/macrobenchmarks/lib/src/web/recorder.dart
@@ -4,6 +4,7 @@
 
 import 'dart:async';
 import 'dart:html' as html;
+import 'dart:js_util' as js_util;
 import 'dart:math' as math;
 import 'dart:ui';
 
@@ -27,6 +28,16 @@ const int _kMeasuredSampleCount = 100;
 /// The total number of samples collected by a benchmark.
 const int kTotalSampleCount = _kWarmUpSampleCount + _kMeasuredSampleCount;
 
+/// A benchmark metric that includes frame-related computations prior to
+/// submitting layer and picture operations to the underlying renderer, such as
+/// HTML and CanvasKit. During this phase we compute transforms, clips, and
+/// other information needed for rendering.
+const String kProfilePrerollFrame = 'preroll_frame';
+
+/// A benchmark metric that includes submitting layer and picture information
+/// to the renderer.
+const String kProfileApplyFrame = 'apply_frame';
+
 /// Measures the amount of time [action] takes.
 Duration timeAction(VoidCallback action) {
   final Stopwatch stopwatch = Stopwatch()..start();
@@ -221,9 +232,9 @@ abstract class SceneBuilderRecorder extends Recorder {
             final Scene scene = sceneBuilder.build();
             profile.record('windowRenderDuration', () {
               window.render(scene);
-            });
-          });
-        });
+            }, reported: false);
+          }, reported: false);
+        }, reported: true);
         endMeasureFrame();
 
         if (profile.shouldContinue()) {
@@ -331,7 +342,7 @@ abstract class WidgetRecorder extends Recorder implements FrameRecorder {
   @mustCallSuper
   void frameDidDraw() {
     endMeasureFrame();
-    profile.addDataPoint('drawFrameDuration', _drawFrameStopwatch.elapsed);
+    profile.addDataPoint('drawFrameDuration', _drawFrameStopwatch.elapsed, reported: true);
 
     if (profile.shouldContinue()) {
       window.scheduleFrame();
@@ -353,12 +364,30 @@ abstract class WidgetRecorder extends Recorder implements FrameRecorder {
     final _RecordingWidgetsBinding binding =
         _RecordingWidgetsBinding.ensureInitialized();
     final Widget widget = createWidget();
+
+    registerEngineBenchmarkValueListener(kProfilePrerollFrame, (num value) {
+      localProfile.addDataPoint(
+        kProfilePrerollFrame,
+        Duration(microseconds: value.toInt()),
+        reported: false,
+      );
+    });
+    registerEngineBenchmarkValueListener(kProfileApplyFrame, (num value) {
+      localProfile.addDataPoint(
+        kProfileApplyFrame,
+        Duration(microseconds: value.toInt()),
+        reported: false,
+      );
+    });
+
     binding._beginRecording(this, widget);
 
     try {
       await _runCompleter.future;
       return localProfile;
     } finally {
+      stopListeningToEngineBenchmarkValues(kProfilePrerollFrame);
+      stopListeningToEngineBenchmarkValues(kProfileApplyFrame);
       _runCompleter = null;
       profile = null;
     }
@@ -421,7 +450,7 @@ abstract class WidgetBuildRecorder extends Recorder implements FrameRecorder {
     // Only record frames that show the widget.
     if (showWidget) {
       endMeasureFrame();
-      profile.addDataPoint('drawFrameDuration', _drawFrameStopwatch.elapsed);
+      profile.addDataPoint('drawFrameDuration', _drawFrameStopwatch.elapsed, reported: true);
     }
 
     if (profile.shouldContinue()) {
@@ -488,11 +517,21 @@ class _WidgetBuildRecorderHostState extends State<_WidgetBuildRecorderHost> {
 /// calculations will only apply to the latest [_kMeasuredSampleCount] data
 /// points.
 class Timeseries {
-  Timeseries(this.name);
+  Timeseries(this.name, this.isReported);
 
   /// The label of this timeseries used for debugging and result inspection.
   final String name;
 
+  /// Whether this timeseries is reported to the benchmark dashboard.
+  ///
+  /// If `true` a new benchmark card is created for the timeseries and is
+  /// visible on the dashboard.
+  ///
+  /// If `false` the data is stored but it does not show up on the dashboard.
+  /// Use unreported metrics for metrics that are useful for manual inspection
+  /// but that are too fine-grained to be useful for tracking on the dashboard.
+  final bool isReported;
+
   /// List of all the values that have been recorded.
   ///
   /// This list has no limit.
@@ -700,14 +739,20 @@ class Profile {
   final Map<String, dynamic> extraData = <String, dynamic>{};
 
   /// Invokes [callback] and records the duration of its execution under [key].
-  Duration record(String key, VoidCallback callback) {
+  Duration record(String key, VoidCallback callback, { @required bool reported }) {
     final Duration duration = timeAction(callback);
-    addDataPoint(key, duration);
+    addDataPoint(key, duration, reported: reported);
     return duration;
   }
 
-  void addDataPoint(String key, Duration duration) {
-    scoreData.putIfAbsent(key, () => Timeseries(key)).add(duration.inMicroseconds.toDouble());
+  /// Adds a timed sample to the timeseries corresponding to [key].
+  ///
+  /// Set [reported] to `true` to report the timeseries to the dashboard UI.
+  ///
+  /// Set [reported] to `false` to store the data, but not show it on the
+  /// dashboard UI.
+  void addDataPoint(String key, Duration duration, { @required bool reported }) {
+    scoreData.putIfAbsent(key, () => Timeseries(key, reported)).add(duration.inMicroseconds.toDouble());
   }
 
   /// Decides whether the data collected so far is sufficient to stop, or
@@ -740,9 +785,16 @@ class Profile {
     };
 
     for (final String key in scoreData.keys) {
-      scoreKeys.add('$key.average');
-      scoreKeys.add('$key.outlierAverage');
       final Timeseries timeseries = scoreData[key];
+
+      if (timeseries.isReported) {
+        scoreKeys.add('$key.average');
+        // Report `outlierRatio` rather than `outlierAverage`, because
+        // the absolute value of outliers is less interesting than the
+        // ratio.
+        scoreKeys.add('$key.outlierRatio');
+      }
+
       final TimeseriesStats stats = timeseries.computeStats();
       json['$key.average'] = stats.average;
       json['$key.outlierAverage'] = stats.outlierAverage;
@@ -958,3 +1010,55 @@ void endMeasureFrame() {
   );
   _currentFrameNumber += 1;
 }
+
+/// A function that receives a benchmark value from the framework.
+typedef EngineBenchmarkValueListener = void Function(num value);
+
+// Maps from a value label name to a listener.
+final Map<String, EngineBenchmarkValueListener> _engineBenchmarkListeners = <String, EngineBenchmarkValueListener>{};
+
+/// Registers a [listener] for engine benchmark values labeled by [name].
+///
+/// If another listener is already registered, overrides it.
+void registerEngineBenchmarkValueListener(String name, EngineBenchmarkValueListener listener) {
+  if (listener == null) {
+    throw ArgumentError(
+      'Listener must not be null. To stop listening to engine benchmark values '
+      'under label "$name", call stopListeningToEngineBenchmarkValues(\'$name\').',
+    );
+  }
+
+  if (_engineBenchmarkListeners.containsKey(name)) {
+    throw StateError(
+      'A listener for "$name" is already registered.\n'
+      'Call `stopListeningToEngineBenchmarkValues` to unregister the previous '
+      'listener before registering a new one.'
+    );
+  }
+
+  if (_engineBenchmarkListeners.isEmpty) {
+    // The first listener is being registered. Register the global listener.
+    js_util.setProperty(html.window, '_flutter_internal_on_benchmark', _dispatchEngineBenchmarkValue);
+  }
+
+  _engineBenchmarkListeners[name] = listener;
+}
+
+/// Stops listening to engine benchmark values under labeled by [name].
+void stopListeningToEngineBenchmarkValues(String name) {
+  _engineBenchmarkListeners.remove(name);
+  if (_engineBenchmarkListeners.isEmpty) {
+    // The last listener unregistered. Remove the global listener.
+    js_util.setProperty(html.window, '_flutter_internal_on_benchmark', null);
+  }
+}
+
+// Dispatches a benchmark value reported by the engine to the relevant listener.
+//
+// If there are no listeners registered for [name], ignores the value.
+void _dispatchEngineBenchmarkValue(String name, double value) {
+  final EngineBenchmarkValueListener listener = _engineBenchmarkListeners[name];
+  if (listener != null) {
+    listener(value);
+  }
+}

Original file line number	Diff line number	Diff line change
`@@ -67,9 +67,9 @@ class BenchPictureRecording extends RawRecorder {`
`67`	`67`	`}`
`68`	`68`	`canvas.restore();`
`69`	`69`	`}`
`70`		`- });`
	`70`	`+ }, reported: true);`
`71`	`71`	`profile.record('estimatePaintBounds', () {`
`72`	`72`	`recorder.endRecording();`
`73`		`- });`
	`73`	`+ }, reported: true);`
`74`	`74`	`}`
`75`	`75`	`}`
Original file line number	Diff line number	Diff line change
`@@ -58,11 +58,6 @@ void _useCanvasText(bool useCanvasText) {`
`58`	`58`	`);`
`59`	`59`	`}`
`60`	`60`
`61`		`-typedef OnBenchmark = void Function(String name, num value);`
`62`		`-void _onBenchmark(OnBenchmark listener) {`
`63`		`- js_util.setProperty(html.window, '_flutter_internal_on_benchmark', listener);`
`64`		`-}`
`65`		`-`
`66`	`61`	`/// Repeatedly lays out a paragraph using the DOM measurement approach.`
`67`	`62`	`///`
`68`	`63`	`/// Creates a different paragraph each time in order to avoid hitting the cache.`
`@@ -132,21 +127,21 @@ class BenchTextLayout extends RawRecorder {`
`132`	`127`	`}) {`
`133`	`128`	`profile.record('$keyPrefix.layout', () {`
`134`	`129`	`paragraph.layout(ui.ParagraphConstraints(width: maxWidth));`
`135`		`- });`
	`130`	`+ }, reported: true);`
`136`	`131`	`profile.record('$keyPrefix.getBoxesForRange', () {`
`137`	`132`	`for (int start = 0; start < text.length; start += 3) {`
`138`	`133`	`for (int end = start + 1; end < text.length; end *= 2) {`
`139`	`134`	`paragraph.getBoxesForRange(start, end);`
`140`	`135`	`}`
`141`	`136`	`}`
`142`		`- });`
	`137`	`+ }, reported: true);`
`143`	`138`	`profile.record('$keyPrefix.getPositionForOffset', () {`
`144`	`139`	`for (double dx = 0.0; dx < paragraph.width; dx += 10.0) {`
`145`	`140`	`for (double dy = 0.0; dy < paragraph.height; dy += 10.0) {`
`146`	`141`	`paragraph.getPositionForOffset(Offset(dx, dy));`
`147`	`142`	`}`
`148`	`143`	`}`
`149`		`- });`
	`144`	`+ }, reported: true);`
`150`	`145`	`}`
`151`	`146`	`}`
`152`	`147`
`@@ -179,7 +174,7 @@ class BenchTextCachedLayout extends RawRecorder {`
`179`	`174`	`final ui.Paragraph paragraph = builder.build();`
`180`	`175`	`profile.record('layout', () {`
`181`	`176`	`paragraph.layout(const ui.ParagraphConstraints(width: double.infinity));`
`182`		`- });`
	`177`	`+ }, reported: true);`
`183`	`178`	`_useCanvasText(null);`
`184`	`179`	`}`
`185`	`180`	`}`
`@@ -242,15 +237,15 @@ class BenchBuildColorsGrid extends WidgetBuildRecorder {`
`242`	`237`	`if (mode == _TestMode.useDomTextLayout) {`
`243`	`238`	`_useCanvasText(false);`
`244`	`239`	`}`
`245`		`- _onBenchmark((String name, num value) {`
	`240`	`+ registerEngineBenchmarkValueListener('text_layout', (num value) {`
`246`	`241`	`_textLayoutMicros += value;`
`247`	`242`	`});`
`248`	`243`	`}`
`249`	`244`
`250`	`245`	`@override`
`251`	`246`	`Future<void> tearDownAll() async {`
`252`	`247`	`_useCanvasText(null);`
`253`		`- _onBenchmark(null);`
	`248`	`+ stopListeningToEngineBenchmarkValues('text_layout');`
`254`	`249`	`}`
`255`	`250`
`256`	`251`	`@override`
`@@ -268,6 +263,7 @@ class BenchBuildColorsGrid extends WidgetBuildRecorder {`
`268`	`263`	`profile.addDataPoint(`
`269`	`264`	`'text_layout',`
`270`	`265`	`Duration(microseconds: _textLayoutMicros.toInt()),`
	`266`	`+ reported: true,`
`271`	`267`	`);`
`272`	`268`	`}`
`273`	`269`	`super.frameDidDraw();`