Split config file handling out of shared.py. NFC. (emscripten-core#12699

)

Move config file handling into its own file.  Also, move a few small
utilities into `utils.py` to avoid circular dependencies.

For a while now shared.py has been way too big and this is step towards
breaking it up and breaking the existing circular dependencies between
python modules that use and also also used by shared.py (cache.py for
example).

em-config.py

@@ -16,17 +16,17 @@
 
 import sys
 import re
-from tools import shared
+from tools import config
 
 
 def main():
   if len(sys.argv) != 2 or \
     not re.match(r"^[\w\W_][\w\W_\d]*$", sys.argv[1]) or \
-    not hasattr(shared, sys.argv[1]):
+    not hasattr(config, sys.argv[1]):
     print('Usage: em-config VAR_NAME', file=sys.stderr)
     exit(1)
 
-  print(getattr(shared, sys.argv[1]))
+  print(getattr(config, sys.argv[1]))
   return 0
 
 

emcc.py

@@ -49,6 +49,7 @@
 from tools import js_manipulation
 from tools import wasm2c
 from tools import webassembly
+from tools import config
 
 if __name__ == '__main__':
   ToolchainProfiler.record_process_start()
@@ -654,8 +655,8 @@ def check_human_readable_list(items):
 
 def make_js_executable(script):
   src = open(script).read()
-  cmd = shared.shlex_join(shared.JS_ENGINE)
-  if not os.path.isabs(shared.JS_ENGINE[0]):
+  cmd = shared.shlex_join(config.JS_ENGINE)
+  if not os.path.isabs(config.JS_ENGINE[0]):
     # TODO: use whereis etc. And how about non-*NIX?
     cmd = '/usr/bin/env -S ' + cmd
   logger.debug('adding `#!` to JavaScript file: %s' % cmd)
@@ -926,7 +927,7 @@ def need_llvm_debug_info():
     # warnings are properly printed during arg parse.
     newargs = diagnostics.capture_warnings(newargs)
 
-    if not shared.CONFIG_FILE:
+    if not config.config_file:
       diagnostics.warning('deprecated', 'Specifying EM_CONFIG as a python literal is deprecated. Please use a file instead.')
 
     for i in range(len(newargs)):
@@ -1947,10 +1948,10 @@ def is_link_flag(flag):
 
     CXX = [shared.CLANG_CXX]
     CC = [shared.CLANG_CC]
-    if shared.COMPILER_WRAPPER:
-      logger.debug('using compiler wrapper: %s', shared.COMPILER_WRAPPER)
-      CXX.insert(0, shared.COMPILER_WRAPPER)
-      CC.insert(0, shared.COMPILER_WRAPPER)
+    if config.COMPILER_WRAPPER:
+      logger.debug('using compiler wrapper: %s', config.COMPILER_WRAPPER)
+      CXX.insert(0, config.COMPILER_WRAPPER)
+      CC.insert(0, config.COMPILER_WRAPPER)
 
     if 'EMMAKEN_COMPILER' in os.environ:
       diagnostics.warning('deprecated', '`EMMAKEN_COMPILER` is deprecated.\n'
@@ -2437,7 +2438,7 @@ def consume_arg():
     elif check_arg('--extern-post-js'):
       options.extern_post_js += open(consume_arg()).read() + '\n'
     elif check_arg('--compiler-wrapper'):
-      shared.COMPILER_WRAPPER = consume_arg()
+      config.COMPILER_WRAPPER = consume_arg()
     elif check_flag('--post-link'):
       options.post_link = True
     elif check_arg('--oformat'):
@@ -2604,7 +2605,7 @@ def consume_arg():
       if os.path.exists(path):
         exit_with_error('File ' + optarg + ' passed to --generate-config already exists!')
       else:
-        shared.generate_config(optarg)
+        config.generate_config(optarg)
       should_exit = True
     # Record USE_PTHREADS setting because it controls whether --shared-memory is passed to lld
     elif arg == '-pthread':

tests/clang_native.py

@@ -5,7 +5,8 @@
 
 import logging
 import os
-from tools.shared import MACOS, WINDOWS, path_from_root, PIPE, run_process, CLANG_CC, CLANG_CXX
+from tools.shared import PIPE, run_process, CLANG_CC, CLANG_CXX
+from tools.utils import MACOS, WINDOWS, path_from_root
 from tools import building
 
 logger = logging.getLogger('clang_native')

tests/jsrun.py

@@ -9,6 +9,7 @@
 from subprocess import Popen, PIPE, CalledProcessError
 
 from tools import shared
+from tools.shared import config
 
 WORKING_ENGINES = {} # Holds all configured engines and whether they work: maps path -> True/False
 
@@ -90,7 +91,7 @@ def run_js(filename, engine=None, args=[],
            stdin=None, stdout=PIPE, stderr=None, cwd=None,
            full_output=False, assert_returncode=0, skip_check=False):
   if not engine:
-    engine = shared.JS_ENGINES[0]
+    engine = config.JS_ENGINES[0]
 
   # We used to support True here but we no longer do.  Assert here just in case.
   assert(type(assert_returncode) == int)

tests/runner.py

@@ -55,12 +55,13 @@
 import jsrun
 import parallel_testsuite
 from jsrun import NON_ZERO
-from tools.shared import EM_CONFIG, TEMP_DIR, EMCC, EMXX, DEBUG
+from tools.config import EM_CONFIG
+from tools.shared import TEMP_DIR, EMCC, EMXX, DEBUG
 from tools.shared import EMSCRIPTEN_TEMP_DIR
-from tools.shared import MACOS, WINDOWS
 from tools.shared import EM_BUILD_VERBOSE
 from tools.shared import asstr, get_canonical_temp_dir, try_delete
-from tools.shared import asbytes, Settings
+from tools.shared import asbytes, Settings, config
+from tools.utils import MACOS, WINDOWS
 from tools import shared, line_endings, building
 
 
@@ -232,24 +233,24 @@ def chdir(dir):
 
 @contextlib.contextmanager
 def js_engines_modify(replacements):
-  """A context manager that updates shared.JS_ENGINES."""
-  original = shared.JS_ENGINES
-  shared.JS_ENGINES = replacements
+  """A context manager that updates config.JS_ENGINES."""
+  original = config.JS_ENGINES
+  config.JS_ENGINES = replacements
   try:
     yield
   finally:
-    shared.JS_ENGINES = original
+    config.JS_ENGINES = original
 
 
 @contextlib.contextmanager
 def wasm_engines_modify(replacements):
-  """A context manager that updates shared.WASM_ENGINES."""
-  original = shared.WASM_ENGINES
-  shared.WASM_ENGINES = replacements
+  """A context manager that updates config.WASM_ENGINES."""
+  original = config.WASM_ENGINES
+  config.WASM_ENGINES = replacements
   try:
     yield
   finally:
-    shared.WASM_ENGINES = original
+    config.WASM_ENGINES = original
 
 
 def ensure_dir(dirname):
@@ -989,7 +990,7 @@ def ccshared(src, linkto=[]):
 
   def filtered_js_engines(self, js_engines=None):
     if js_engines is None:
-      js_engines = shared.JS_ENGINES
+      js_engines = config.JS_ENGINES
     for engine in js_engines:
       assert type(engine) == list
     for engine in self.banned_js_engines:
@@ -1046,7 +1047,7 @@ def _build_and_run(self, filename, expected_output, args=[], output_nicerizer=No
     if self.get_setting('STANDALONE_WASM'):
       # TODO once standalone wasm support is more stable, apply use_all_engines
       # like with js engines, but for now as we bring it up, test in all of them
-      wasm_engines = shared.WASM_ENGINES
+      wasm_engines = config.WASM_ENGINES
       if len(wasm_engines) == 0:
         logger.warning('no wasm engine was found to run the standalone part of this test')
       engines += wasm_engines
@@ -1656,8 +1657,8 @@ def open_make_err(mode='r'):
 
 
 def check_js_engines():
-  working_engines = list(filter(jsrun.check_engine, shared.JS_ENGINES))
-  if len(working_engines) < len(shared.JS_ENGINES):
+  working_engines = list(filter(jsrun.check_engine, config.JS_ENGINES))
+  if len(working_engines) < len(config.JS_ENGINES):
     print('Not all the JS engines in JS_ENGINES appears to work.')
     exit(1)
 

tests/test_benchmark.py

@@ -18,8 +18,8 @@
 import clang_native
 import jsrun
 import runner
-from tools.shared import run_process, path_from_root, SPIDERMONKEY_ENGINE, LLVM_ROOT, V8_ENGINE, PIPE, try_delete, EMCC
-from tools import shared, building
+from tools.shared import run_process, path_from_root, PIPE, try_delete, EMCC, config
+from tools import building
 
 # standard arguments for timing:
 # 0: no runtime, just startup
@@ -188,7 +188,7 @@ def build(self, parent, filename, args, shared_args, emcc_args, native_args, nat
     self.filename = filename
     self.old_env = os.environ
     os.environ = self.env.copy()
-    llvm_root = self.env.get('LLVM') or LLVM_ROOT
+    llvm_root = self.env.get('LLVM') or config.LLVM_ROOT
     if lib_builder:
       env_init = self.env.copy()
       # Note that we need to pass in all the flags here because some build
@@ -356,11 +356,11 @@ def cleanup(self):
   # NativeBenchmarker('gcc',   'gcc',    'g++')
 ]
 
-if V8_ENGINE and V8_ENGINE in shared.JS_ENGINES:
+if config.V8_ENGINE and config.V8_ENGINE in config.JS_ENGINES:
   # avoid the baseline compiler running, because it adds a lot of noise
   # (the nondeterministic time it takes to get to the full compiler ends up
   # mattering as much as the actual benchmark)
-  aot_v8 = V8_ENGINE + ['--no-liftoff']
+  aot_v8 = config.V8_ENGINE + ['--no-liftoff']
   default_v8_name = os.environ.get('EMBENCH_NAME') or 'v8'
   benchmarkers += [
     EmscriptenBenchmarker(default_v8_name, aot_v8),
@@ -372,7 +372,7 @@ def cleanup(self):
       # CheerpBenchmarker('cheerp-v8-wasm', aot_v8),
     ]
 
-if SPIDERMONKEY_ENGINE and SPIDERMONKEY_ENGINE in shared.JS_ENGINES:
+if config.SPIDERMONKEY_ENGINE and config.SPIDERMONKEY_ENGINE in config.JS_ENGINES:
   # TODO: ensure no baseline compiler is used, see v8
   benchmarkers += [
     # EmscriptenBenchmarker('sm', SPIDERMONKEY_ENGINE),
@@ -382,7 +382,7 @@ def cleanup(self):
       # CheerpBenchmarker('cheerp-sm-wasm', SPIDERMONKEY_ENGINE),
     ]
 
-if shared.NODE_JS and shared.NODE_JS in shared.JS_ENGINES:
+if config.NODE_JS and config.NODE_JS in config.JS_ENGINES:
   benchmarkers += [
     # EmscriptenBenchmarker('Node.js', shared.NODE_JS),
   ]
@@ -408,7 +408,7 @@ def setUpClass(cls):
         fingerprint.append('sm: ' + [line for line in run_process(['hg', 'tip'], stdout=PIPE).stdout.splitlines() if 'changeset' in line][0])
     except Exception:
       pass
-    fingerprint.append('llvm: ' + LLVM_ROOT)
+    fingerprint.append('llvm: ' + config.LLVM_ROOT)
     print('Running Emscripten benchmarks... [ %s ]' % ' | '.join(fingerprint))
 
   # avoid depending on argument reception from the commandline

tests/test_browser.py

@@ -25,8 +25,8 @@
 from tools import building
 from tools import shared
 from tools import system_libs
-from tools.shared import PYTHON, EMCC, WINDOWS, FILE_PACKAGER, PIPE, V8_ENGINE
-from tools.shared import try_delete
+from tools.shared import PYTHON, EMCC, WINDOWS, FILE_PACKAGER, PIPE
+from tools.shared import try_delete, config
 
 
 def test_chunked_synchronous_xhr_server(support_byte_ranges, chunkSize, data, checksum, port):
@@ -4887,7 +4887,7 @@ def test_zzz_zzz_4GB(self):
     # test that we can allocate in the 2-4GB range, if we enable growth and
     # set the max appropriately
     self.emcc_args += ['-O2', '-s', 'ALLOW_MEMORY_GROWTH', '-s', 'MAXIMUM_MEMORY=4GB']
-    self.do_run_in_out_file_test('tests', 'browser', 'test_4GB.cpp', js_engines=[V8_ENGINE])
+    self.do_run_in_out_file_test('tests', 'browser', 'test_4GB.cpp', js_engines=[config.V8_ENGINE])
 
   @no_firefox('no 4GB support yet')
   def test_zzz_zzz_2GB_fail(self):
@@ -4900,7 +4900,7 @@ def test_zzz_zzz_2GB_fail(self):
     # test that growth doesn't go beyond 2GB without the max being set for that,
     # and that we can catch an allocation failure exception for that
     self.emcc_args += ['-O2', '-s', 'ALLOW_MEMORY_GROWTH', '-s', 'MAXIMUM_MEMORY=2GB']
-    self.do_run_in_out_file_test('tests', 'browser', 'test_2GB_fail.cpp', js_engines=[V8_ENGINE])
+    self.do_run_in_out_file_test('tests', 'browser', 'test_2GB_fail.cpp', js_engines=[config.V8_ENGINE])
 
   @no_firefox('no 4GB support yet')
   def test_zzz_zzz_4GB_fail(self):
@@ -4913,7 +4913,7 @@ def test_zzz_zzz_4GB_fail(self):
     # test that we properly report an allocation error that would overflow over
     # 4GB.
     self.emcc_args += ['-O2', '-s', 'ALLOW_MEMORY_GROWTH', '-s', 'MAXIMUM_MEMORY=4GB', '-s', 'ABORTING_MALLOC=0']
-    self.do_run_in_out_file_test('tests', 'browser', 'test_4GB_fail.cpp', js_engines=[V8_ENGINE])
+    self.do_run_in_out_file_test('tests', 'browser', 'test_4GB_fail.cpp', js_engines=[config.V8_ENGINE])
 
   @unittest.skip("only run this manually, to test for race conditions")
   @parameterized({