Perf test with SkSL warm-up (flutter#56638)

liyuqian · web-flow · commit 9dad2d98dfde · 2020-06-06T14:43:02.000-07:00
diff --git a/dev/devicelab/bin/tasks/cubic_bezier_perf_sksl_warmup__timeline_summary.dart b/dev/devicelab/bin/tasks/cubic_bezier_perf_sksl_warmup__timeline_summary.dart
@@ -0,0 +1,14 @@
+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:async';
+
+import 'package:flutter_devicelab/tasks/perf_tests.dart';
+import 'package:flutter_devicelab/framework/adb.dart';
+import 'package:flutter_devicelab/framework/framework.dart';
+
+Future<void> main() async {
+  deviceOperatingSystem = DeviceOperatingSystem.android;
+  await task(createCubicBezierPerfSkSLWarmupTest());
+}
diff --git a/dev/devicelab/lib/tasks/perf_tests.dart b/dev/devicelab/lib/tasks/perf_tests.dart
@@ -7,6 +7,7 @@ import 'dart:convert' show LineSplitter, json, utf8;
 import 'dart:io';
 import 'dart:math' as math;
 
+import 'package:flutter_devicelab/tasks/track_widget_creation_enabled_task.dart';
 import 'package:meta/meta.dart';
 import 'package:path/path.dart' as path;
 
@@ -82,12 +83,20 @@ TaskFunction createCubicBezierPerfTest() {
   ).run;
 }
 
+TaskFunction createCubicBezierPerfSkSLWarmupTest() {
+  return PerfTestWithSkSL(
+    '${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
+    'test_driver/cubic_bezier_perf.dart',
+    'cubic_bezier_perf',
+  ).run;
+}
+
 TaskFunction createBackdropFilterPerfTest({bool needsMeasureCpuGpu = false}) {
   return PerfTest(
     '${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
     'test_driver/backdrop_filter_perf.dart',
     'backdrop_filter_perf',
-    needsMeasureCpuGPu: needsMeasureCpuGpu,
+    needsMeasureCpuGpu: needsMeasureCpuGpu,
   ).run;
 }
 
@@ -96,7 +105,7 @@ TaskFunction createPostBackdropFilterPerfTest({bool needsMeasureCpuGpu = false})
     '${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
     'test_driver/post_backdrop_filter_perf.dart',
     'post_backdrop_filter_perf',
-    needsMeasureCpuGPu: needsMeasureCpuGpu,
+    needsMeasureCpuGpu: needsMeasureCpuGpu,
   ).run;
 }
 
@@ -105,7 +114,7 @@ TaskFunction createSimpleAnimationPerfTest({bool needsMeasureCpuGpu = false}) {
     '${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
     'test_driver/simple_animation_perf.dart',
     'simple_animation_perf',
-    needsMeasureCpuGPu: needsMeasureCpuGpu,
+    needsMeasureCpuGpu: needsMeasureCpuGpu,
   ).run;
 }
 
@@ -114,7 +123,7 @@ TaskFunction createAnimatedPlaceholderPerfTest({bool needsMeasureCpuGpu = false}
     '${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
     'test_driver/animated_placeholder_perf.dart',
     'animated_placeholder_perf',
-    needsMeasureCpuGPu: needsMeasureCpuGpu,
+    needsMeasureCpuGpu: needsMeasureCpuGpu,
   ).run;
 }
 
@@ -266,7 +275,7 @@ class PerfTest {
     this.testDirectory,
     this.testTarget,
     this.timelineFileName, {
-    this.needsMeasureCpuGPu = false,
+    this.needsMeasureCpuGpu = false,
     this.testDriver,
   });
 
@@ -279,9 +288,19 @@ class PerfTest {
   /// The test file to run on the host.
   final String testDriver;
   /// Whether to collect CPU and GPU metrics.
-  final bool needsMeasureCpuGPu;
+  final bool needsMeasureCpuGpu;
 
   Future<TaskResult> run() {
+    return internalRun();
+  }
+
+  @protected
+  Future<TaskResult> internalRun({
+      bool keepRunning = false,
+      bool cacheSkSL = false,
+      String existingApp,
+      String writeSkslFileName,
+  }) {
     return inDirectory<TaskResult>(testDirectory, () async {
       final Device device = await devices.workingDevice;
       await device.unlock();
@@ -295,7 +314,13 @@ class PerfTest {
         '-t',
         testTarget,
         if (testDriver != null)
-          '--driver', testDriver,
+          ...<String>['--driver', testDriver],
+        if (existingApp != null)
+          ...<String>['--use-existing-app', existingApp],
+        if (writeSkslFileName != null)
+          ...<String>['--write-sksl-on-exit', writeSkslFileName],
+        if (cacheSkSL) '--cache-sksl',
+        if (keepRunning) '--keep-app-running',
         '-d',
         deviceId,
       ]);
@@ -310,7 +335,7 @@ class PerfTest {
         );
       }
 
-      if (needsMeasureCpuGPu) {
+      if (needsMeasureCpuGpu) {
         await inDirectory<void>('$testDirectory/build', () async {
           data.addAll(await measureIosCpuGpu(deviceId: deviceId));
         });
@@ -328,13 +353,117 @@ class PerfTest {
         'average_vsync_transitions_missed',
         '90th_percentile_vsync_transitions_missed',
         '99th_percentile_vsync_transitions_missed',
-        if (needsMeasureCpuGPu) 'cpu_percentage',
-        if (needsMeasureCpuGPu) 'gpu_percentage',
+        if (needsMeasureCpuGpu) 'cpu_percentage',
+        if (needsMeasureCpuGpu) 'gpu_percentage',
       ]);
     });
   }
 }
 
+class PerfTestWithSkSL extends PerfTest {
+  PerfTestWithSkSL(
+    String testDirectory,
+    String testTarget,
+    String timelineFileName, {
+    bool needsMeasureCpuGpu = false,
+    String testDriver,
+  }) : super(
+    testDirectory,
+    testTarget,
+    timelineFileName,
+    needsMeasureCpuGpu: needsMeasureCpuGpu,
+    testDriver: testDriver,
+  );
+
+  @override
+  Future<TaskResult> run() async {
+    return inDirectory<TaskResult>(testDirectory, () async {
+      // Some initializations
+      _device = await devices.workingDevice;
+      _flutterPath = path.join(flutterDirectory.path, 'bin', 'flutter');
+
+      // Prepare the SkSL by running the driver test.
+      await _generateSkSL();
+
+      // Build the app with SkSL artifacts and run that app
+      final String observatoryUri = await _buildAndRun();
+
+      // Attach to the running app and run the final driver test to get metrics.
+      final TaskResult result = await internalRun(
+        existingApp: observatoryUri,
+      );
+
+      _runProcess.kill();
+      await _runProcess.exitCode;
+
+      return result;
+    });
+  }
+
+  Future<void> _generateSkSL() async {
+    await super.internalRun(
+      keepRunning: true,
+      cacheSkSL: true,
+      writeSkslFileName: _skslJsonFileName,
+    );
+  }
+
+  // Return the VMService URI.
+  Future<String> _buildAndRun() async {
+    await flutter('build', options: <String>[
+      // TODO(liyuqian): also supports iOS once https://github.com/flutter/flutter/issues/53115 is fully closed.
+      'apk',
+      '--profile',
+      '--bundle-sksl-path', _skslJsonFileName,
+      '-t', testTarget,
+    ]);
+
+    if (File(_vmserviceFileName).existsSync()) {
+      File(_vmserviceFileName).deleteSync();
+    }
+
+    _runProcess = await startProcess(
+      _flutterPath,
+      <String>[
+        'run',
+        '--verbose',
+        '--profile',
+        '-d', _device.deviceId,
+        '-t', testTarget,
+        '--endless-trace-buffer',
+        '--use-application-binary',
+        '$testDirectory/build/app/outputs/flutter-apk/app-profile.apk',
+        '--vmservice-out-file', _vmserviceFileName,
+      ],
+    );
+
+    final Stream<List<int>> broadcastOut = _runProcess.stdout.asBroadcastStream();
+    _forwardStream(broadcastOut, 'run stdout');
+    _forwardStream(_runProcess.stderr, 'run stderr');
+
+    final File file = await waitForFile(_vmserviceFileName);
+    return file.readAsStringSync();
+  }
+
+  String get _skslJsonFileName => '$testDirectory/flutter_01.sksl.json';
+  String get _vmserviceFileName => '$testDirectory/$_kVmserviceOutFileName';
+
+  Stream<String> _transform(Stream<List<int>> stream) =>
+      stream.transform<String>(utf8.decoder).transform<String>(const LineSplitter());
+
+  void _forwardStream(Stream<List<int>> stream, String label) {
+    _transform(stream).listen((String line) {
+      print('$label: $line');
+    });
+  }
+
+  String _flutterPath;
+  Device _device;
+  Process _runProcess;
+
+  static const String _kVmserviceOutFileName = 'vmservice.out';
+}
+
 /// Measures how long it takes to compile a Flutter app to JavaScript and how
 /// big the compiled code is.
 class WebCompileTest {
diff --git a/dev/devicelab/manifest.yaml b/dev/devicelab/manifest.yaml
@@ -168,6 +168,14 @@ tasks:
     stage: devicelab
     required_agent_capabilities: ["mac/android"]
 
+  cubic_bezier_perf_sksl_warmup__timeline_summary:
+    description: >
+      Measures the runtime performance of cubic bezier animations on Android
+      with SkSL shader warm-up.
+    stage: devicelab
+    required_agent_capabilities: ["mac/android"]
+    flaky: true
+
   backdrop_filter_perf__timeline_summary:
     description: >
       Measures the runtime performance of backdrop filter blurs on Android.
