Add bootstrap.py script which takes care of post-checkout tasks (em…

…scripten-core#19736)

This script is only present is git checkout of emscripten and not in
emsdk or other packaged versions.

When run with no arguments this script ensures that all the
post-checkout steps required by emscripten have been run and are
up-to-date with the current checkout.  It does this by comparing
`.stamp` files in the `out` directory with the timestamps of inputs
files such as `package.json`.

When `emcc` is run, and bootstrap.py is present it is run in `check`
mode which will cause the compiler to error if the one or more of the
bootstrap commands needs to be run (e.g. if package.json was modified).

.circleci/config.yml

@@ -57,6 +57,10 @@ commands:
             # In order make our external version of emscripten use the emsdk
             # config we need to explicitly set EM_CONFIG here.
             echo "export EM_CONFIG=~/emsdk/.emscripten" >> $BASH_ENV
+  bootstrap:
+    description: "bootstrap"
+    steps:
+      - run: ./bootstrap
   npm-install:
     description: "npm ci"
     steps:
@@ -156,6 +160,7 @@ commands:
             echo "final .emscripten:"
             cat ~/emsdk/.emscripten
       - emsdk-env
+      - bootstrap
       - npm-install
   build-libs:
     description: "Build all libraries"
@@ -214,6 +219,7 @@ commands:
           name: submodule update
           command: git submodule update --init
       - emsdk-env
+      - bootstrap
       - npm-install
       - pip-install
   upload-test-results:

ChangeLog.md

@@ -20,6 +20,10 @@ See docs/process.md for more on how version tagging works.
 
 3.1.48 (in development)
 -----------------------
+- A new top-level `bootstrap` script was added.  This script is for emscripten
+  developers and helps take a care of post-checkout tasks such as `npm install`.
+  If this script needs to be run (e.g. becuase package.json was changed, emcc
+  will exit with an error. (#19736)
 
 3.1.47 - 10/09/23
 -----------------

Makefile

@@ -7,7 +7,7 @@ dist: $(DISTFILE)
 install:
 	@rm -rf $(DESTDIR)
 	./tools/install.py $(DESTDIR)
-	npm install --production --prefix $(DESTDIR)
+	npm install --omit=dev --prefix $(DESTDIR)
 
 # Create an distributable archive of emscripten suitable for use
 # by end users. This archive excludes node_modules as it can include native

bootstrap

@@ -0,0 +1,35 @@
+#!/bin/sh
+# Copyright 2020 The Emscripten Authors.  All rights reserved.
+# Emscripten is available under two separate licenses, the MIT license and the
+# University of Illinois/NCSA Open Source License.  Both these licenses can be
+# found in the LICENSE file.
+#
+# Entry point for running python scripts on UNIX systems.
+#
+# Automatically generated by `create_entry_points.py`; DO NOT EDIT.
+#
+# To make modifications to this file, edit `tools/run_python.sh` and then run
+# `tools/maint/create_entry_points.py`
+
+# $PYTHON -E will not ignore _PYTHON_SYSCONFIGDATA_NAME an internal
+# of cpython used in cross compilation via setup.py.
+unset _PYTHON_SYSCONFIGDATA_NAME
+
+if [ -z "$PYTHON" ]; then
+  PYTHON=$EMSDK_PYTHON
+fi
+
+if [ -z "$PYTHON" ]; then
+  PYTHON=$(command -v python3 2> /dev/null)
+fi
+
+if [ -z "$PYTHON" ]; then
+  PYTHON=$(command -v python 2> /dev/null)
+fi
+
+if [ -z "$PYTHON" ]; then
+  echo 'unable to find python in $PATH'
+  exit 1
+fi
+
+exec "$PYTHON" -E "$0.py" "$@"

bootstrap.bat

@@ -0,0 +1,86 @@
+:: Entry point for running python scripts on windows systems.
+::
+:: Automatically generated by `create_entry_points.py`; DO NOT EDIT.
+::
+:: To make modifications to this file, edit `tools/run_python.bat` and then run
+:: `tools/maint/create_entry_points.py`
+
+:: N.b. In Windows .bat scripts, the ':' character cannot appear inside any if () blocks,
+:: or there will be a parsing error.
+
+:: All env. vars specified in this file are to be local only to this script.
+@setlocal
+:: -E will not ignore _PYTHON_SYSCONFIGDATA_NAME an internal
+:: of cpython used in cross compilation via setup.py.
+@set _PYTHON_SYSCONFIGDATA_NAME=
+@set EM_PY=%EMSDK_PYTHON%
+@if "%EM_PY%"=="" (
+  set EM_PY=python
+)
+
+:: Work around Windows bug https://github.com/microsoft/terminal/issues/15212 : If this
+:: script is invoked via enclosing the invocation in quotes via PATH lookup, then %~f0 and
+:: %~dp0 expansions will not work.
+:: So first try if %~dp0 might work, and if not, manually look up this script from PATH.
+@if exist %~f0 (
+  set MYDIR=%~dp0
+  goto FOUND_MYDIR
+)
+@for %%I in (%~n0.bat) do (
+  @if exist %%~$PATH:I (
+    set MYDIR=%%~dp$PATH:I
+  ) else (
+    echo Fatal Error! Due to a Windows bug, we are unable to locate the path to %~n0.bat.
+    echo To help this issue, try removing unnecessary quotes in the invocation of emcc,
+    echo or add Emscripten directory to PATH.
+    echo See github.com/microsoft/terminal/issues/15212 and
+    echo github.com/emscripten-core/emscripten/issues/19207 for more details.
+  )
+)
+:FOUND_MYDIR
+
+:: Python Windows bug https://bugs.python.org/issue34780: If this script was invoked via a
+:: shared stdin handle from the parent process, and that parent process stdin handle is in
+:: a certain state, running python.exe might hang here. To work around this, if
+:: EM_WORKAROUND_PYTHON_BUG_34780 is defined, invoke python with '< NUL' stdin to avoid
+:: sharing the parent's stdin handle to it, avoiding the hang.
+
+:: On Windows 7, the compiler batch scripts are observed to exit with a non-zero errorlevel,
+:: even when the python executable above did succeed and quit with errorlevel 0 above.
+:: On Windows 8 and newer, this issue has not been observed. It is possible that this
+:: issue is related to the above python bug, but this has not been conclusively confirmed,
+:: so using a separate env. var EM_WORKAROUND_WIN7_BAD_ERRORLEVEL_BUG to enable the known
+:: workaround this issue, which is to explicitly quit the calling process with the previous
+:: errorlevel from the above command.
+
+:: Also must use goto to jump to the command dispatch, since we cannot invoke emcc from
+:: inside a if() block, because if a cmdline param would contain a char '(' or ')', that
+:: would throw off the parsing of the cmdline arg.
+@if "%EM_WORKAROUND_PYTHON_BUG_34780%"=="" (
+  @if "%EM_WORKAROUND_WIN7_BAD_ERRORLEVEL_BUG%"=="" (
+    goto NORMAL
+  ) else (
+    goto NORMAL_EXIT
+  )
+) else (
+  @if "%EM_WORKAROUND_WIN7_BAD_ERRORLEVEL_BUG%"=="" (
+    goto MUTE_STDIN
+  ) else (
+    goto MUTE_STDIN_EXIT
+  )
+)
+
+:NORMAL_EXIT
+@"%EM_PY%" -E "%MYDIR%%~n0.py" %*
+@exit %ERRORLEVEL%
+
+:MUTE_STDIN
+@"%EM_PY%" -E "%MYDIR%%~n0.py" %* < NUL
+@exit /b %ERRORLEVEL%
+
+:MUTE_STDIN_EXIT
+@"%EM_PY%" -E "%MYDIR%%~n0.py" %* < NUL
+@exit %ERRORLEVEL%
+
+:NORMAL
+@"%EM_PY%" -E "%MYDIR%%~n0.py" %*

bootstrap.py

@@ -0,0 +1,66 @@
+#!/usr/bin/env python3
+"""Bootstrap script for emscripten developers / git users.
+
+After checking out emscripten there are certain steps that need to be
+taken before it can be used.  This script enumerates and automates
+these steps and is able to run just the steps that are needed based
+on the timestamps of various input files (kind of like a dumb version
+of a Makefile).
+"""
+import argparse
+import os
+import shutil
+import sys
+
+__rootdir__ = os.path.dirname(os.path.abspath(__file__))
+sys.path.insert(0, __rootdir__)
+
+STAMP_DIR = os.path.join(__rootdir__, 'out')
+
+from tools import shared, utils
+
+actions = [
+  ('npm packages', 'package.json', [shutil.which('npm'), 'ci']),
+  # TODO(sbc): Remove the checked in entry point files and have them
+  # built on demand by this step.
+  ('create entry points', 'tools/maint/create_entry_points.py', [sys.executable, 'tools/maint/create_entry_points.py']),
+  ('git submodules', 'test/third_party/posixtestsuite/', [shutil.which('git'), 'submodule', 'update', '--init']),
+]
+
+
+def get_stamp_file(action_name):
+  return os.path.join(STAMP_DIR, action_name.replace(' ', '_') + '.stamp')
+
+
+def check():
+  for name, filename, _ in actions:
+    stamp_file = get_stamp_file(name)
+    filename = utils.path_from_root(filename)
+    if not os.path.exists(stamp_file) or os.path.getmtime(filename) > os.path.getmtime(stamp_file):
+      utils.exit_with_error(f'emscripten setup is not complete ("{name}" is out-of-date). Run bootstrap.py to update')
+
+
+def main(args):
+  parser = argparse.ArgumentParser(description=__doc__)
+  parser.add_argument('-v', '--verbose', action='store_true', help='verbose', default=False)
+  parser.add_argument('-n', '--dry-run', action='store_true', help='dry run', default=False)
+  args = parser.parse_args()
+
+  for name, filename, cmd in actions:
+    stamp_file = get_stamp_file(name)
+    filename = utils.path_from_root(filename)
+    if os.path.exists(stamp_file) and os.path.getmtime(filename) <= os.path.getmtime(stamp_file):
+      print('Up-to-date: %s' % name)
+      continue
+    print('Out-of-date: %s' % name)
+    if args.dry_run:
+      print(' (skipping: dry run) -> %s' % ' '.join(cmd))
+      return
+    print(' -> %s' % ' '.join(cmd))
+    shared.run_process(cmd, cwd=utils.path_from_root())
+    utils.safe_ensure_dirs(STAMP_DIR)
+    utils.write_file(stamp_file, 'Timestamp file created by bootstrap.py')
+
+
+if __name__ == '__main__':
+  main(sys.argv[1:])

emcc.py

@@ -59,6 +59,14 @@
 
 logger = logging.getLogger('emcc')
 
+# In git checkouts of emscripten `bootstrap.py` exists to run post-checkout
+# steps.  In packaged versions (e.g. emsdk) this file does not exist (because
+# it is excluded in tools/install.py) and these steps are assumed to have been
+# run already.
+if os.path.exists(utils.path_from_root('.git')) and os.path.exists(utils.path_from_root('bootstrap.py')):
+  import bootstrap
+  bootstrap.check()
+
 # endings = dot + a suffix, compare against result of shared.suffix()
 C_ENDINGS = ['.c', '.i']
 CXX_ENDINGS = ['.cppm', '.pcm', '.cpp', '.cxx', '.cc', '.c++', '.CPP', '.CXX', '.C', '.CC', '.C++', '.ii']

site/source/docs/building_from_source/index.rst

@@ -4,11 +4,23 @@
 Building Emscripten from Source
 ===============================
 
-Building Emscripten yourself is an alternative to getting binaries using the emsdk.
-
-Emscripten's core codebase, which is in the main "emscripten" repo, does not need to be compiled (it uses Python for most of the scripting that glues together all the tools). What do need to be compiled are LLVM (which in particular provides clang and wasm-ld) and Binaryen. After compiling them, simply edit the ``.emscripten`` file to point to the right place for each of those tools (if the file doesn't exist yet, run ``emcc`` for the first time).
-
-Get the ``main`` branches, or check the `Packaging <https://github.com/emscripten-core/emscripten/blob/main/docs/packaging.md>`_ instructions to identify precise commits in existing releases.
+Building Emscripten yourself is an alternative to getting binaries using the
+emsdk.
+
+Emscripten itself is written in Python and JavaScript so it does not need to be
+compiled.  However, after checkout you will need to perform various steps
+before it can be used (e.g. ``npm install``).  The ``bootstrap`` script in the
+top level of the repository takes care of running these steps and ``emcc`` will
+error out if it detects that ``bootstrap`` needs to be run.
+
+In addition to the main emscripten repository you will also need to checkout
+and build LLVM and Binaryen (as detailed below).  After compiling these, you
+will need to edit your ``.emscripten`` file to point to their corresponding
+locations.
+
+Use the ``main`` branches of each of these repositories, or check the `Packaging
+<https://github.com/emscripten-core/emscripten/blob/main/docs/packaging.md>`_
+instructions to identify precise commits used in a specific release.
 
 
 Building LLVM

test/test_sanity.py

@@ -747,3 +747,19 @@ def test_binaryen_version(self):
 
     make_fake_tool(self.in_dir('fake', 'bin', 'wasm-opt'), '70')
     self.check_working([EMCC, test_file('hello_world.c')], 'unexpected binaryen version: 70 (expected ')
+
+  def test_bootstrap(self):
+    restore_and_set_up()
+    self.run_process([EMCC, test_file('hello_world.c')])
+
+    # Touching package.json should cause compiler to fail with bootstrap message
+    Path(utils.path_from_root('package.json')).touch()
+    err = self.expect_fail([EMCC, test_file('hello_world.c')])
+    self.assertContained('emcc: error: emscripten setup is not complete ("npm packages" is out-of-date). Run bootstrap.py to update', err)
+
+    # Running bootstrap.py should fix that
+    bootstrap = shared.bat_suffix(shared.path_from_root('bootstrap'))
+    self.run_process([bootstrap])
+
+    # Now the compiler should work again
+    self.run_process([EMCC, test_file('hello_world.c')])

tools/install.py

@@ -19,12 +19,14 @@
 
 EXCLUDES = [os.path.normpath(x) for x in '''
 test/third_party
+tools/maint
 site
 node_modules
 Makefile
 .git
 cache
 cache.lock
+bootstrap.py
 '''.split()]
 
 EXCLUDE_PATTERNS = '''
@@ -82,7 +84,6 @@ def copy_emscripten(target):
 
 
 def main():
-
   parser = argparse.ArgumentParser(description=__doc__)
   parser.add_argument('-v', '--verbose', action='store_true', help='verbose',
                       default=int(os.environ.get('EMCC_DEBUG', '0')))

tools/maint/create_entry_points.py

@@ -23,6 +23,7 @@
 '''.split()
 
 entry_points = '''
+bootstrap
 emar
 embuilder
 emcmake