Write binaries directly with s2wasm (emscripten-core#6323)

jgravelle-google · web-flow · commit b238cc06abfa · 2018-03-09T08:23:48.000-08:00
* Read invoke_funcs from metadata now

* Initial support for binary-only s2wasm

* Fix source map test expectations

* Remove commented-out debug copy

* Remove unused wast_temp

* Actually call self.is_wasm_backend
diff --git a/emcc.py b/emcc.py
@@ -1625,9 +1625,7 @@ def get_final():
       if shared.Settings.WASM_BACKEND:
         # we also received wast and wasm at this stage
         temp_basename = unsuffixed(final)
-        wast_temp = temp_basename + '.wast'
         wasm_temp = temp_basename + '.wasm'
-        shutil.move(wast_temp, wasm_text_target)
         shutil.move(wasm_temp, wasm_binary_target)
         open(wasm_text_target + '.mappedGlobals', 'w').write('{}') # no need for mapped globals for now, but perhaps some day
         if options.debug_level >= 4 and not shared.Settings.EXPERIMENTAL_USE_LLD:
diff --git a/emscripten.py b/emscripten.py
@@ -1732,10 +1732,9 @@ def emscript_wasm_backend(infile, settings, outfile, libraries=None, compiler_en
   if libraries is None: libraries = []
 
   if shared.Settings.EXPERIMENTAL_USE_LLD:
-    wast, metadata = build_wasm_lld(temp_files, infile, outfile, settings, DEBUG)
+    wasm, metadata = build_wasm_lld(temp_files, infile, outfile, settings, DEBUG)
   else:
-    wast = build_wasm(temp_files, infile, outfile, settings, DEBUG)
-    metadata = read_metadata_wast(wast, DEBUG)
+    wasm, metadata = build_wasm(temp_files, infile, outfile, settings, DEBUG)
 
   # optimize syscalls
 
@@ -1789,7 +1788,7 @@ def emscript_wasm_backend(infile, settings, outfile, libraries=None, compiler_en
   outfile.write(pre)
   pre = None
 
-  invoke_funcs = read_wast_invoke_imports(wast)
+  invoke_funcs = metadata.get('invokeFuncs', [])
   # List of function signatures used in jsCall functions, e.g.['v', 'vi']
   jscall_sigs = metadata.get('jsCallFuncType', [])
 
@@ -1827,33 +1826,29 @@ def build_wasm(temp_files, infile, outfile, settings, DEBUG):
 
     assert shared.Settings.BINARYEN_ROOT, 'need BINARYEN_ROOT config set so we can use Binaryen s2wasm on the backend output'
     basename = shared.unsuffixed(outfile.name)
-    wast = basename + '.wast'
     wasm = basename + '.wasm'
-    s2wasm_args = create_s2wasm_args(temp_s)
-    if DEBUG:
-      logging.debug('emscript: binaryen s2wasm: ' + ' '.join(s2wasm_args))
-      t = time.time()
-      #s2wasm_args += ['--debug']
-    shared.check_call(s2wasm_args, stdout=open(wast, 'w'))
-    # Also convert wasm text to binary
-    wasm_as_args = [os.path.join(shared.Settings.BINARYEN_ROOT, 'bin', 'wasm-as'),
-                    wast, '-o', wasm]
+    metadata_file = basename + '.metadata'
+    s2wasm_args = create_s2wasm_args(temp_s, wasm)
     if settings['DEBUG_LEVEL'] >= 2 or settings['PROFILING_FUNCS']:
-      wasm_as_args += ['-g']
+      s2wasm_args += ['-g']
       if settings['DEBUG_LEVEL'] >= 4:
-        wasm_as_args += ['--source-map=' + wasm + '.map']
+        s2wasm_args += ['--source-map=' + wasm + '.map']
         if not settings['SOURCE_MAP_BASE']:
           logging.warn("Wasm source map won't be usable in a browser without --source-map-base")
         else:
-          wasm_as_args += ['--source-map-url=' + settings['SOURCE_MAP_BASE'] + os.path.basename(settings['WASM_BINARY_FILE']) + '.map']
-    logging.debug('  emscript: binaryen wasm-as: ' + ' '.join(wasm_as_args))
-    shared.check_call(wasm_as_args)
+          s2wasm_args += ['--source-map-url=' + settings['SOURCE_MAP_BASE'] + os.path.basename(settings['WASM_BINARY_FILE']) + '.map']
+    if DEBUG:
+      logging.debug('emscript: binaryen s2wasm: ' + ' '.join(s2wasm_args))
+      t = time.time()
+      #s2wasm_args += ['--debug']
+    shared.check_call(s2wasm_args, stdout=open(metadata_file, 'w'))
+
+  metadata = create_metadata_wasm(open(metadata_file).read(), DEBUG)
 
   if DEBUG:
     logging.debug('  emscript: binaryen s2wasm took %s seconds' % (time.time() - t))
     t = time.time()
-    shutil.copyfile(wast, os.path.join(shared.CANONICAL_TEMP_DIR, 'emcc-s2wasm-output.wast'))
-  return wast
+  return wasm, metadata
 
 
 def build_wasm_lld(temp_files, infile, outfile, settings, DEBUG):
@@ -1882,8 +1877,8 @@ def debug_copy(src, dst):
     debug_copy(temp_o, 'emcc-llvm-backend-output.o')
 
     basename = shared.unsuffixed(outfile.name)
-    wast = basename + '.wast'
     wasm = basename + '.wasm'
+    metadata_file = basename + '.metadata'
     base_wasm = basename + '.lld.wasm'
 
     libc_rt_lib = shared.Cache.get('wasm_libc_rt.a', wasm_rt_fail('wasm_libc_rt.a'), 'a')
@@ -1907,20 +1902,15 @@ def debug_copy(src, dst):
       t = time.time()
     debug_copy(base_wasm, 'base_wasm.wasm')
 
-    # TODO: We currently read exports from the wast in order to generate
-    # metadata. So we emit text here so we can parse wast from python.
-    shared.check_call([wasm_emscripten_finalize, base_wasm, '-o', wast, '-S',
+    shared.check_call([wasm_emscripten_finalize, base_wasm, '-o', wasm,
                        '--global-base=%s' % shared.Settings.GLOBAL_BASE,
                        ('--emscripten-reserved-function-pointers=%d' %
-                        shared.Settings.RESERVED_FUNCTION_POINTERS)])
-    debug_copy(wast, 'lld-emscripten-output.wast')
+                        shared.Settings.RESERVED_FUNCTION_POINTERS)],
+                      stdout=open(metadata_file, 'w'))
 
-    shared.check_call([wasm_as, wast, '-o', wasm])
-    debug_copy(wasm, 'lld-emscripten-output.wasm')
+    metadata = create_metadata_wasm(open(metadata_file).read(), DEBUG)
 
-    metadata = read_metadata_wast(wast, DEBUG)
-
-  return wast, metadata
+  return wasm, metadata
 
 
 def read_metadata_wast(wast, DEBUG):
@@ -2028,17 +2018,6 @@ def create_em_js(forwarded_json, metadata):
   return em_js_funcs
 
 
-def read_wast_invoke_imports(wast):
-  invoke_funcs = []
-  for line in open(wast).readlines():
-    if line.strip().startswith('(import '):
-      parts = line.split()
-      func_name = parts[2][1:-1]
-      if func_name.startswith('invoke_'):
-        invoke_funcs.append(func_name)
-  return invoke_funcs
-
-
 def create_sending_wasm(invoke_funcs, jscall_sigs, forwarded_json, metadata,
                         settings):
   basic_funcs = ['abort', 'assert', 'enlargeMemory', 'getTotalMemory']
@@ -2158,13 +2137,13 @@ def wrapped():
     raise Exception('Expected {} to already be built'.format(archive_file))
   return wrapped
 
-def create_s2wasm_args(temp_s):
+def create_s2wasm_args(temp_s, wasm):
   compiler_rt_lib = shared.Cache.get('wasm_compiler_rt.a', wasm_rt_fail('wasm_compiler_rt.a'), 'a')
   libc_rt_lib = shared.Cache.get('wasm_libc_rt.a', wasm_rt_fail('wasm_libc_rt.a'), 'a')
 
   s2wasm_path = os.path.join(shared.Settings.BINARYEN_ROOT, 'bin', 's2wasm')
 
-  args = [s2wasm_path, temp_s, '--emscripten-glue']
+  args = [s2wasm_path, temp_s, '-o', wasm, '--emscripten-glue', '--emit-binary']
   args += ['--global-base=%d' % shared.Settings.GLOBAL_BASE]
   args += ['--initial-memory=%d' % shared.Settings.TOTAL_MEMORY]
   args += ['--allow-memory-growth'] if shared.Settings.ALLOW_MEMORY_GROWTH else []
@@ -2194,6 +2173,7 @@ def load_metadata(metadata_raw):
     'initializers': [],
     'exports': [],
     'emJsFuncs': {},
+    'invokeFuncs': [],
   }
 
   for k, v in metadata_json.items():
diff --git a/tests/test_core.py b/tests/test_core.py
@@ -6929,8 +6929,18 @@ def encode_utf8(data):
         # the file attribute is optional, but if it is present it needs to refer
         # the output file.
         self.assertPathsIdentical(map_referent, data['file'])
-      assert len(data['sources']) == 1, data['sources']
-      self.assertPathsIdentical(src_filename, data['sources'][0])
+      if not self.is_wasm_backend():
+        assert len(data['sources']) == 1, data['sources']
+        self.assertPathsIdentical(src_filename, data['sources'][0])
+      else:
+        # Wasm backend currently adds every file linked as part of compiler-rt
+        # to the 'sources' field.
+        # TODO(jgravelle): when LLD is the wasm-backend default, make sure it
+        # emits only the files we have lines for.
+        assert len(data['sources']) > 1, data['sources']
+        normalized_srcs = [src.replace('\\', '/') for src in data['sources']]
+        normalized_filename = src_filename.replace('\\', '/')
+        assert normalized_filename in normalized_srcs, "Source file not found"
       if hasattr(data, 'sourcesContent'):
         # the sourcesContent attribute is optional, but if it is present it
         # needs to containt valid source text.
