Add build script invalidation and snapshotting logic (flutter#28866)

Jonah Williams · web-flow · commit dd944994186a · 2019-03-06T15:16:16.000-08:00
diff --git a/packages/flutter_tools/lib/src/build_runner/build_runner.dart b/packages/flutter_tools/lib/src/build_runner/build_runner.dart
@@ -12,13 +12,14 @@ import 'package:build_daemon/data/build_status.dart' as build;
 import 'package:build_daemon/client.dart';
 import 'package:meta/meta.dart';
 import 'package:yaml/yaml.dart';
+import 'package:crypto/crypto.dart' show md5;
 
 import '../artifacts.dart';
+import '../base/common.dart';
 import '../base/file_system.dart';
 import '../base/io.dart';
 import '../base/logger.dart';
 import '../base/process_manager.dart';
-import '../cache.dart';
 import '../codegen.dart';
 import '../convert.dart';
 import '../dart/pub.dart';
@@ -53,24 +54,23 @@ class BuildRunner extends CodeGenerator {
     final String sdkRoot = artifacts.getArtifactPath(Artifact.flutterPatchedSdkPath);
     final String engineDartBinaryPath = artifacts.getArtifactPath(Artifact.engineDartBinary);
     final String packagesPath = flutterProject.packagesFile.absolute.path;
-    final String buildScript = flutterProject
+    final String buildSnapshot = flutterProject
         .dartTool
         .childDirectory('build')
         .childDirectory('entrypoint')
-        .childFile('build.dart')
+        .childFile('build.dart.snapshot')
         .path;
     final String scriptPackagesPath = flutterProject
         .dartTool
         .childDirectory('flutter_tool')
         .childFile('.packages')
         .path;
-    final String dartPath = fs.path.join(Cache.flutterRoot, 'bin', 'cache', 'dart-sdk', 'bin', 'dart');
     final Status status = logger.startProgress('running builders...', timeout: null);
     try {
       final Process buildProcess = await processManager.start(<String>[
-        dartPath,
+        engineDartBinaryPath,
         '--packages=$scriptPackagesPath',
-        buildScript,
+        buildSnapshot,
         'build',
         '--skip-build-script-check',
         '--define', 'flutter_build|kernel=disabled=$disableKernelGeneration',
@@ -95,7 +95,6 @@ class BuildRunner extends CodeGenerator {
           .transform(utf8.decoder)
           .transform(const LineSplitter())
           .listen(printError);
-      await buildProcess.exitCode;
     } finally {
       status.stop();
     }
@@ -127,57 +126,84 @@ class BuildRunner extends CodeGenerator {
     return CodeGenerationResult(packagesFile, dillFile);
   }
 
-  @override
-  Future<void> invalidateBuildScript() async {
-    final FlutterProject flutterProject = await FlutterProject.current();
-    final File buildScript = flutterProject.dartTool
-        .absolute
-        .childDirectory('flutter_tool')
-        .childFile('build.dart');
-    if (!buildScript.existsSync()) {
-      return;
-    }
-    await buildScript.delete();
-  }
-
   @override
   Future<void> generateBuildScript() async {
     final FlutterProject flutterProject = await FlutterProject.current();
-    final String generatedDirectory = fs.path.join(flutterProject.dartTool.path, 'flutter_tool');
-    final String resultScriptPath = fs.path.join(flutterProject.dartTool.path, 'build', 'entrypoint', 'build.dart');
-    if (fs.file(resultScriptPath).existsSync()) {
-      return;
+    final Directory entrypointDirectory = fs.directory(fs.path.join(flutterProject.dartTool.path, 'build', 'entrypoint'));
+    final Directory generatedDirectory = fs.directory(fs.path.join(flutterProject.dartTool.path, 'flutter_tool'));
+    final File buildScript = entrypointDirectory.childFile('build.dart');
+    final File buildSnapshot = entrypointDirectory.childFile('build.dart.snapshot');
+    final File scriptIdFile = entrypointDirectory.childFile('id');
+    final File syntheticPubspec = generatedDirectory.childFile('pubspec.yaml');
+
+    // Check if contents of builders changed. If so, invalidate build script
+    // and regnerate.
+    final YamlMap builders = await flutterProject.builders;
+    final List<int> appliedBuilderDigest = _produceScriptId(builders);
+    if (scriptIdFile.existsSync() && buildSnapshot.existsSync()) {
+      final List<int> previousAppliedBuilderDigest = scriptIdFile.readAsBytesSync();
+      bool digestsAreEqual = false;
+      if (appliedBuilderDigest.length == previousAppliedBuilderDigest.length) {
+        digestsAreEqual = true;
+        for (int i = 0; i < appliedBuilderDigest.length; i++) {
+          if (appliedBuilderDigest[i] != previousAppliedBuilderDigest[i]) {
+            digestsAreEqual = false;
+            break;
+          }
+        }
+      }
+      if (digestsAreEqual) {
+        return;
+      }
+    }
+    // Clean-up all existing artifacts.
+    if (flutterProject.dartTool.existsSync()) {
+      flutterProject.dartTool.deleteSync(recursive: true);
     }
     final Status status = logger.startProgress('generating build script...', timeout: null);
     try {
-      fs.directory(generatedDirectory).createSync(recursive: true);
-
-      final File syntheticPubspec = fs.file(fs.path.join(generatedDirectory, 'pubspec.yaml'));
+      generatedDirectory.createSync(recursive: true);
+      entrypointDirectory.createSync(recursive: true);
+      flutterProject.dartTool.childDirectory('build').childDirectory('generated').createSync(recursive: true);
       final StringBuffer stringBuffer = StringBuffer();
 
       stringBuffer.writeln('name: flutter_tool');
       stringBuffer.writeln('dependencies:');
       final YamlMap builders = await flutterProject.builders;
       if (builders != null) {
         for (String name in builders.keys) {
-          final YamlNode node = builders[name];
+          final Object node = builders[name];
           stringBuffer.writeln('  $name: $node');
         }
       }
       stringBuffer.writeln('  build_runner: any');
       stringBuffer.writeln('  flutter_build:');
       stringBuffer.writeln('    sdk: flutter');
-      await syntheticPubspec.writeAsString(stringBuffer.toString());
+      syntheticPubspec.writeAsStringSync(stringBuffer.toString());
 
       await pubGet(
         context: PubContext.pubGet,
-        directory: generatedDirectory,
+        directory: generatedDirectory.path,
         upgrade: false,
         checkLastModified: false,
       );
+      if (!scriptIdFile.existsSync()) {
+        scriptIdFile.createSync(recursive: true);
+      }
+      scriptIdFile.writeAsBytesSync(appliedBuilderDigest);
       final PackageGraph packageGraph = PackageGraph.forPath(syntheticPubspec.parent.path);
       final BuildScriptGenerator buildScriptGenerator = const BuildScriptGeneratorFactory().create(flutterProject, packageGraph);
       await buildScriptGenerator.generateBuildScript();
+      final ProcessResult result = await processManager.run(<String>[
+        artifacts.getArtifactPath(Artifact.engineDartBinary),
+        '--snapshot=${buildSnapshot.path}',
+        '--snapshot-kind=app-jit',
+        '--packages=${fs.path.join(generatedDirectory.path, '.packages')}',
+        buildScript.path,
+      ]);
+      if (result.exitCode != 0) {
+        throwToolExit('Error generating build_script snapshot: ${result.stderr}');
+      }
     } finally {
       status.stop();
     }
@@ -200,25 +226,23 @@ class BuildRunner extends CodeGenerator {
     final String sdkRoot = artifacts.getArtifactPath(Artifact.flutterPatchedSdkPath);
     final String engineDartBinaryPath = artifacts.getArtifactPath(Artifact.engineDartBinary);
     final String packagesPath = flutterProject.packagesFile.absolute.path;
-    final String buildScript = flutterProject
+    final File buildSnapshot = flutterProject
         .dartTool
         .childDirectory('build')
         .childDirectory('entrypoint')
-        .childFile('build.dart')
-        .path;
+        .childFile('build.dart.snapshot');
     final String scriptPackagesPath = flutterProject
         .dartTool
         .childDirectory('flutter_tool')
         .childFile('.packages')
         .path;
-    final String dartPath = fs.path.join(Cache.flutterRoot, 'bin', 'cache', 'dart-sdk', 'bin', 'dart');
     final Status status = logger.startProgress('starting build daemon...', timeout: null);
     BuildDaemonClient buildDaemonClient;
     try {
       final List<String> command = <String>[
-        dartPath,
+        engineDartBinaryPath,
         '--packages=$scriptPackagesPath',
-        buildScript,
+        buildSnapshot.path,
         'daemon',
          '--skip-build-script-check',
         '--define', 'flutter_build|kernel=disabled=false',
@@ -279,3 +303,14 @@ class _BuildRunnerCodegenDaemon implements CodegenDaemon {
     buildDaemonClient.startBuild();
   }
 }
+
+// Sorts the builders by name and produces a hashcode of the resulting iterable.
+List<int> _produceScriptId(YamlMap builders) {
+  if (builders == null || builders.isEmpty) {
+    return md5.convert(<int>[]).bytes;
+  }
+  final List<String> orderedBuilders = builders.keys
+    .cast<String>()
+    .toList()..sort();
+  return md5.convert(orderedBuilders.join('').codeUnits).bytes;
+}
diff --git a/packages/flutter_tools/lib/src/codegen.dart b/packages/flutter_tools/lib/src/codegen.dart
@@ -81,12 +81,6 @@ abstract class CodeGenerator {
     List<String> extraFrontEndOptions = const <String>[],
   });
 
-  /// Invalidates a generated build script by deleting it.
-  ///
-  /// Must be called any time a pubspec file update triggers a corresponding change
-  /// in .packages.
-  Future<void> invalidateBuildScript();
-
   // Generates a synthetic package under .dart_tool/flutter_tool which is in turn
   // used to generate a build script.
   Future<void> generateBuildScript();
@@ -113,11 +107,6 @@ class UnsupportedCodeGenerator extends CodeGenerator {
     throw UnsupportedError('build_runner is not currently supported.');
   }
 
-  @override
-  Future<void> invalidateBuildScript() {
-    throw UnsupportedError('build_runner is not currently supported.');
-  }
-
   @override
   Future<CodegenDaemon> daemon({
     String mainPath,
