Use subprocess.run + universal_newlines=True (#5843)

* explicit utf-8 environment

...as the Ubuntu images on Docker defaults to LANG=C (ascii)

* python 3 style run_process utility

* split by \n when universal_newlines

* use run_process in many places

Author: saschanaz

Dockerfile

@@ -1,7 +1,7 @@
 # For travis
 FROM buildpack-deps:xenial
 SHELL ["/bin/bash", "-c"]
-ENV DEBIAN_FRONTEND noninteractive
+ENV DEBIAN_FRONTEND=noninteractive LANG=C.UTF-8
 RUN mkdir -p /root/emscripten/
 COPY . /root/emscripten/
 

emcc.py

@@ -31,7 +31,7 @@
 import os, sys, shutil, tempfile, subprocess, shlex, time, re, logging
 from subprocess import PIPE
 from tools import shared, jsrun, system_libs
-from tools.shared import execute, suffix, unsuffixed, unsuffixed_basename, WINDOWS, safe_move
+from tools.shared import execute, suffix, unsuffixed, unsuffixed_basename, WINDOWS, safe_move, run_process
 from tools.response_file import read_response_file
 import tools.line_endings
 
@@ -386,8 +386,8 @@ def run():
     args = [x for x in sys.argv if x != '--cflags']
     with misc_temp_files.get_file(suffix='.o') as temp_target:
       input_file = 'hello_world.c'
-      out, err = subprocess.Popen([shared.PYTHON] + args + [shared.path_from_root('tests', input_file), '-c', '-o', temp_target], stderr=subprocess.PIPE, env=debug_env).communicate()
-      lines = [x for x in err.split(os.linesep) if shared.CLANG_CC in x and input_file in x]
+      err = run_process([shared.PYTHON] + args + [shared.path_from_root('tests', input_file), '-c', '-o', temp_target], stderr=subprocess.PIPE, env=debug_env).stderr
+      lines = [x for x in err.split('\n') if shared.CLANG_CC in x and input_file in x]
       line = re.search('running: (.*)', lines[0]).group(1)
       parts = shlex.split(line.replace('\\', '\\\\'))
       parts = [x for x in parts if x != '-c' and x != '-o' and input_file not in x and temp_target not in x and '-emit-llvm' not in x]

emscripten.py

@@ -120,7 +120,7 @@ def compile_js(infile, settings, temp_files, DEBUG):
       logging.debug('emscript: llvm backend: ' + ' '.join(backend_args))
       t = time.time()
     with ToolchainProfiler.profile_block('emscript_llvm_backend'):
-      shared.jsrun.timeout_run(subprocess.Popen(backend_args, stdout=subprocess.PIPE), note_args=backend_args)
+      shared.jsrun.timeout_run(subprocess.Popen(backend_args, stdout=subprocess.PIPE, universal_newlines=True), note_args=backend_args)
     if DEBUG:
       logging.debug('  emscript: llvm backend took %s seconds' % (time.time() - t))
 

tools/bisect_pair.py

@@ -37,7 +37,7 @@ def run_code(name):
 
 # Calculate diff chunks
 print('diffing')
-diff = Popen(['diff', '-U', '5', 'left', 'right'], stdout=PIPE).communicate()[0].split('\n')
+diff = run_process(['diff', '-U', '5', 'left', 'right'], stdout=PIPE).stdout.split('\n')
 pre_diff = diff[:2]
 diff = diff[2:]
 
@@ -81,7 +81,7 @@ def run_code(name):
     found = mid
     break
 
-critical = Popen(['diff', '-U', '5', 'middle' + str(mid-1), 'middle' + str(mid)], stdout=PIPE).communicate()[0]
+critical = run_process(['diff', '-U', '5', 'middle' + str(mid-1), 'middle' + str(mid)], stdout=PIPE).stdout
 c = open('critical.diff', 'w')
 c.write(critical)
 c.close()

tools/bisect_pair_wast.py

@@ -38,7 +38,7 @@ def run_code(name):
 
 # Calculate diff chunks
 print('diffing')
-diff = Popen(['diff', '-U', '5', 'left', 'right'], stdout=PIPE).communicate()[0].split('\n')
+diff = run_process(['diff', '-U', '5', 'left', 'right'], stdout=PIPE).stdout.split('\n')
 pre_diff = diff[:2]
 diff = diff[2:]
 
@@ -82,7 +82,7 @@ def run_code(name):
     found = mid
     break
 
-critical = Popen(['diff', '-U', '5', 'middle' + str(mid-1), 'middle' + str(mid)], stdout=PIPE).communicate()[0]
+critical = run_process(['diff', '-U', '5', 'middle' + str(mid-1), 'middle' + str(mid)], stdout=PIPE).stdout
 c = open('critical.diff', 'w')
 c.write(critical)
 c.close()

tools/ctor_evaller.py

@@ -247,7 +247,7 @@ def read_and_delete(filename):
     err_file = config.get_temp_files().get('.err').name
     out_file_handle = open(out_file, 'w')
     err_file_handle = open(err_file, 'w')
-    proc = subprocess.Popen(shared.NODE_JS + [temp_file], stdout=out_file_handle, stderr=err_file_handle)
+    proc = subprocess.Popen(shared.NODE_JS + [temp_file], stdout=out_file_handle, stderr=err_file_handle, universal_newlines=True)
     try:
       shared.jsrun.timeout_run(proc, timeout=10, full_output=True, throw_on_failure=False)
     except Exception as e:
@@ -289,7 +289,7 @@ def eval_ctors_wasm(js, wasm_file, num):
   if debug_info:
     cmd += ['-g']
   shared.logging.debug('wasm ctor cmd: ' + str(cmd))
-  out, err = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
+  err = shared.run_process(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE).stderr
   num_successful = err.count('success on')
   shared.logging.debug(err)
   if len(ctors) == num_successful:

tools/duplicate_function_eliminator.py

@@ -20,13 +20,13 @@ def process_shell(js_engine, shell, equivalentfn_hash_info=None):
     f.write(equivalentfn_hash_info)
     f.close()
 
-    (output,error) = subprocess.Popen(js_engine +
+    proc = shared.run_process(js_engine +
         [DUPLICATE_FUNCTION_ELIMINATOR, temp_file, '--use-hash-info', '--no-minimize-whitespace'],
-        stdout=subprocess.PIPE,stderr=subprocess.PIPE).communicate()
-  assert len(output) > 0
-  assert len(error) == 0
+        stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+  assert len(proc.stdout) > 0
+  assert len(proc.stderr) == 0
 
-  return output
+  return proc.stdout
 
 def run_on_chunk(command):
   try:
@@ -45,15 +45,13 @@ def run_on_chunk(command):
 
     if shared.EM_BUILD_VERBOSE_LEVEL >= 3: print('run_on_chunk: ' + str(command), file=sys.stderr)
 
-    proc = subprocess.Popen(command, stdout=subprocess.PIPE)
-    output = proc.communicate()[0]
+    proc = shared.run_process(command, stdout=subprocess.PIPE)
+    output = proc.stdout
     assert proc.returncode == 0, 'Error in optimizer (return code ' + str(proc.returncode) + '): ' + output
     assert len(output) > 0 and not output.startswith('Assertion failed'), 'Error in optimizer: ' + output
     filename = temp_files.get(os.path.basename(filename) + '.jo' + file_suffix).name
 
-    # Important to write out in binary mode, because the data we are writing contains Windows line endings '\r\n' because it was PIPED from console.
-    # Otherwise writing \r\n to ascii mode file will result in Windows amplifying \n to \r\n, generating bad \r\r\n line endings.
-    f = open(filename, 'wb')
+    f = open(filename, 'w')
     f.write(output)
     f.close()
     if DEBUG and not shared.WINDOWS: print('.', file=sys.stderr) # Skip debug progress indicator on Windows, since it doesn't buffer well with multiple threads printing to console.

tools/ffdb.py

@@ -176,18 +176,18 @@ def print_applist(applist, running_app_manifests, print_removable):
 
 def adb_devices():
   try:
-    devices = subprocess.check_output([ADB, 'devices'])
+    devices = run_process([ADB, 'devices'], check=True, stdout=subprocess.PIPE).stdout
     devices = devices.strip().split('\n')[1:]
     devices = [x.strip().split('\t') for x in devices]
     return devices
   except Exception as e:
     return []
 
 def b2g_get_prefs_filename():
-  return subprocess.check_output([ADB, 'shell', 'echo', '-n', '/data/b2g/mozilla/*.default/prefs.js'])
+  return run_process([ADB, 'shell', 'echo', '-n', '/data/b2g/mozilla/*.default/prefs.js'], check=True, stdout=subprocess.PIPE).stdout
 
 def b2g_get_prefs_data():
-  return subprocess.check_output([ADB, 'shell', 'cat', '/data/b2g/mozilla/*.default/prefs.js'])
+  return run_process([ADB, 'shell', 'cat', '/data/b2g/mozilla/*.default/prefs.js'], check=True, stdout=subprocess.PIPE).stdout
 
 def b2g_get_pref(sub):
   prefs_data = b2g_get_prefs_data().split('\n')

tools/file_packager.py

@@ -365,7 +365,7 @@ def was_seen(name):
 
       # guess at format. this lets us tell crunch to not try to be clever and use odd formats like DXT5_AGBR
       try:
-        format = Popen(['file', file_['srcpath']], stdout=PIPE).communicate()[0]
+        format = run_process(['file', file_['srcpath']], stdout=PIPE).stdout
         if 'DXT5' in format:
           format = ['-dxt5']
         elif 'DXT1' in format:

tools/js_optimizer.py

@@ -246,11 +246,11 @@ def minify_shell(self, shell, minify_whitespace, source_map=False):
       f.write('// EXTRA_INFO:' + json.dumps(self.serialize()))
       f.close()
 
-      output = subprocess.Popen(self.js_engine +
+      output = shared.run_process(self.js_engine +
           [JS_OPTIMIZER, temp_file, 'minifyGlobals', 'noPrintMetadata'] +
           (['minifyWhitespace'] if minify_whitespace else []) +
           (['--debug'] if source_map else []),
-          stdout=subprocess.PIPE).communicate()[0]
+          stdout=subprocess.PIPE).stdout
 
     assert len(output) > 0 and not output.startswith('Assertion failed'), 'Error in js optimizer: ' + output
     #print >> sys.stderr, "minified SHELL 3333333333333333", output, "\n44444444444444444444"
@@ -290,14 +290,12 @@ def run_on_chunk(command):
       print('running js optimizer command', ' '.join([c if c != filename else saved for c in command]), file=sys.stderr)
       shutil.copyfile(filename, os.path.join(shared.get_emscripten_temp_dir(), saved))
     if shared.EM_BUILD_VERBOSE_LEVEL >= 3: print('run_on_chunk: ' + str(command), file=sys.stderr)
-    proc = subprocess.Popen(command, stdout=subprocess.PIPE)
-    output = proc.communicate()[0]
+    proc = shared.run_process(command, stdout=subprocess.PIPE)
+    output = proc.stdout
     assert proc.returncode == 0, 'Error in optimizer (return code ' + str(proc.returncode) + '): ' + output
     assert len(output) > 0 and not output.startswith('Assertion failed'), 'Error in optimizer: ' + output
     filename = temp_files.get(os.path.basename(filename) + '.jo.js').name
-    # Important to write out in binary mode, because the data we are writing contains Windows line endings '\r\n' because it was PIPED from console.
-    # Otherwise writing \r\n to ascii mode file will result in Windows amplifying \n to \r\n, generating bad \r\r\n line endings.
-    f = open(filename, 'wb')
+    f = open(filename, 'w')
     f.write(output)
     f.close()
     if DEBUG and not shared.WINDOWS: print('.', file=sys.stderr) # Skip debug progress indicator on Windows, since it doesn't buffer well with multiple threads printing to console.