Implement some more of the wasi API (#12704)

In order to test our WASI API implementation I've added
`wasi-test-suite` as submodule.  This submodule is only
needed for testing.

Author: sbc100

.gitmodules

@@ -4,3 +4,6 @@
 [submodule "test/third_party/googletest"]
 	path = test/third_party/googletest
 	url = https://github.com/google/googletest
+[submodule "test/third_party/wasi-test-suite"]
+  path = test/third_party/wasi-test-suite
+  url = https://github.com/khronosproject/wasi-test-suite

emcc.py

@@ -1137,7 +1137,7 @@ def package_files(options, target):
   if options.preload_files:
     # Preloading files uses --pre-js code that runs before the module is loaded.
     file_code = shared.check_call(cmd, stdout=PIPE).stdout
-    js_manipulation.add_files_pre_js(options.pre_js, file_code)
+    js_manipulation.add_files_pre_js(settings.PRE_JS_FILES, file_code)
   else:
     # Otherwise, we are embedding files, which does not require --pre-js code,
     # and instead relies on a static constrcutor to populate the filesystem.
@@ -1311,9 +1311,6 @@ def run(args):
   if len(options.preload_files) or len(options.embed_files):
     linker_arguments += package_files(options, target)
 
-  settings.PRE_JS_FILES = [os.path.abspath(f) for f in options.pre_js]
-  settings.POST_JS_FILES = [os.path.abspath(f) for f in options.post_js]
-
   if options.oformat == OFormat.OBJECT:
     logger.debug(f'link_to_object: {linker_arguments} -> {target}')
     building.link_to_object(linker_arguments, target)
@@ -1891,7 +1888,7 @@ def phase_linker_setup(options, state, newargs):
     # Requesting both Wasm and Wasm2JS support
     settings.WASM2JS = 1
 
-  if (options.oformat == OFormat.WASM or settings.PURE_WASI) and not settings.SIDE_MODULE:
+  if options.oformat == OFormat.WASM and not settings.SIDE_MODULE:
     # if the output is just a wasm file, it will normally be a standalone one,
     # as there is no JS. an exception are side modules, as we can't tell at
     # compile time whether JS will be involved or not - the main module may
@@ -1902,6 +1899,10 @@ def phase_linker_setup(options, state, newargs):
   if settings.LZ4:
     settings.EXPORTED_RUNTIME_METHODS += ['LZ4']
 
+  if settings.PURE_WASI:
+    settings.STANDALONE_WASM = 1
+    settings.WASM_BIGINT = 1
+
   if settings.WASM2C:
     # wasm2c only makes sense with standalone wasm - there will be no JS,
     # just wasm and then C
@@ -2906,6 +2907,9 @@ def get_full_import_name(name):
     # in standalone mode, crt1 will call the constructors from inside the wasm
     settings.REQUIRED_EXPORTS.append('__wasm_call_ctors')
 
+  settings.PRE_JS_FILES = [os.path.abspath(f) for f in options.pre_js]
+  settings.POST_JS_FILES = [os.path.abspath(f) for f in options.post_js]
+
   return target, wasm_target
 
 

emscripten.py

@@ -77,8 +77,9 @@ def optimize_syscalls(declares):
     # without filesystem support, it doesn't matter what syscalls need
     settings.SYSCALLS_REQUIRE_FILESYSTEM = 0
   else:
+    # TODO(sbc): Find a better way to identify wasi syscalls
     syscall_prefixes = ('__syscall_', 'fd_')
-    syscalls = {d for d in declares if d.startswith(syscall_prefixes)}
+    syscalls = {d for d in declares if d.startswith(syscall_prefixes) or d in ['path_open']}
     # check if the only filesystem syscalls are in: close, ioctl, llseek, write
     # (without open, etc.. nothing substantial can be done, so we can disable
     # extra filesystem support in that case)

src/generated_struct_info32.json

@@ -394,10 +394,49 @@
         "__WASI_CLOCKID_PROCESS_CPUTIME_ID": 2,
         "__WASI_CLOCKID_REALTIME": 0,
         "__WASI_CLOCKID_THREAD_CPUTIME_ID": 3,
+        "__WASI_FDFLAGS_APPEND": 1,
+        "__WASI_FDFLAGS_DSYNC": 2,
+        "__WASI_FDFLAGS_NONBLOCK": 4,
+        "__WASI_FDFLAGS_RSYNC": 8,
+        "__WASI_FDFLAGS_SYNC": 16,
         "__WASI_FILETYPE_CHARACTER_DEVICE": 2,
         "__WASI_FILETYPE_DIRECTORY": 3,
         "__WASI_FILETYPE_REGULAR_FILE": 4,
-        "__WASI_FILETYPE_SYMBOLIC_LINK": 7
+        "__WASI_FILETYPE_SYMBOLIC_LINK": 7,
+        "__WASI_OFLAGS_CREAT": 1,
+        "__WASI_OFLAGS_DIRECTORY": 2,
+        "__WASI_OFLAGS_EXCL": 4,
+        "__WASI_OFLAGS_TRUNC": 8,
+        "__WASI_PREOPENTYPE_DIR": 0,
+        "__WASI_RIGHTS_FD_ADVISE": 128,
+        "__WASI_RIGHTS_FD_ALLOCATE": 256,
+        "__WASI_RIGHTS_FD_DATASYNC": 1,
+        "__WASI_RIGHTS_FD_FDSTAT_SET_FLAGS": 8,
+        "__WASI_RIGHTS_FD_FILESTAT_GET": 2097152,
+        "__WASI_RIGHTS_FD_FILESTAT_SET_SIZE": 4194304,
+        "__WASI_RIGHTS_FD_FILESTAT_SET_TIMES": 8388608,
+        "__WASI_RIGHTS_FD_READ": 2,
+        "__WASI_RIGHTS_FD_READDIR": 16384,
+        "__WASI_RIGHTS_FD_SEEK": 4,
+        "__WASI_RIGHTS_FD_SYNC": 16,
+        "__WASI_RIGHTS_FD_TELL": 32,
+        "__WASI_RIGHTS_FD_WRITE": 64,
+        "__WASI_RIGHTS_PATH_CREATE_DIRECTORY": 512,
+        "__WASI_RIGHTS_PATH_CREATE_FILE": 1024,
+        "__WASI_RIGHTS_PATH_FILESTAT_GET": 262144,
+        "__WASI_RIGHTS_PATH_FILESTAT_SET_SIZE": 524288,
+        "__WASI_RIGHTS_PATH_FILESTAT_SET_TIMES": 1048576,
+        "__WASI_RIGHTS_PATH_LINK_SOURCE": 2048,
+        "__WASI_RIGHTS_PATH_LINK_TARGET": 4096,
+        "__WASI_RIGHTS_PATH_OPEN": 8192,
+        "__WASI_RIGHTS_PATH_READLINK": 32768,
+        "__WASI_RIGHTS_PATH_REMOVE_DIRECTORY": 33554432,
+        "__WASI_RIGHTS_PATH_RENAME_SOURCE": 65536,
+        "__WASI_RIGHTS_PATH_RENAME_TARGET": 131072,
+        "__WASI_RIGHTS_PATH_SYMLINK": 16777216,
+        "__WASI_RIGHTS_PATH_UNLINK_FILE": 67108864,
+        "__WASI_RIGHTS_POLL_FD_READWRITE": 134217728,
+        "__WASI_RIGHTS_SOCK_SHUTDOWN": 268435456
     },
     "structs": {
         "EmscriptenBatteryEvent": {
@@ -1236,6 +1275,26 @@
             "fs_rights_base": 8,
             "fs_rights_inheriting": 16
         },
+        "__wasi_filestat_t": {
+            "__size__": 64,
+            "atim": 40,
+            "ctim": 56,
+            "dev": 0,
+            "filetype": 16,
+            "ino": 8,
+            "mtim": 48,
+            "nlink": 24,
+            "size": 32
+        },
+        "__wasi_prestat_dir_t": {
+            "__size__": 4,
+            "pr_name_len": 0
+        },
+        "__wasi_prestat_t": {
+            "__size__": 8,
+            "pr_type": 0,
+            "u": 4
+        },
         "addrinfo": {
             "__size__": 32,
             "ai_addr": 20,

src/generated_struct_info64.json

@@ -394,10 +394,49 @@
         "__WASI_CLOCKID_PROCESS_CPUTIME_ID": 2,
         "__WASI_CLOCKID_REALTIME": 0,
         "__WASI_CLOCKID_THREAD_CPUTIME_ID": 3,
+        "__WASI_FDFLAGS_APPEND": 1,
+        "__WASI_FDFLAGS_DSYNC": 2,
+        "__WASI_FDFLAGS_NONBLOCK": 4,
+        "__WASI_FDFLAGS_RSYNC": 8,
+        "__WASI_FDFLAGS_SYNC": 16,
         "__WASI_FILETYPE_CHARACTER_DEVICE": 2,
         "__WASI_FILETYPE_DIRECTORY": 3,
         "__WASI_FILETYPE_REGULAR_FILE": 4,
-        "__WASI_FILETYPE_SYMBOLIC_LINK": 7
+        "__WASI_FILETYPE_SYMBOLIC_LINK": 7,
+        "__WASI_OFLAGS_CREAT": 1,
+        "__WASI_OFLAGS_DIRECTORY": 2,
+        "__WASI_OFLAGS_EXCL": 4,
+        "__WASI_OFLAGS_TRUNC": 8,
+        "__WASI_PREOPENTYPE_DIR": 0,
+        "__WASI_RIGHTS_FD_ADVISE": 128,
+        "__WASI_RIGHTS_FD_ALLOCATE": 256,
+        "__WASI_RIGHTS_FD_DATASYNC": 1,
+        "__WASI_RIGHTS_FD_FDSTAT_SET_FLAGS": 8,
+        "__WASI_RIGHTS_FD_FILESTAT_GET": 2097152,
+        "__WASI_RIGHTS_FD_FILESTAT_SET_SIZE": 4194304,
+        "__WASI_RIGHTS_FD_FILESTAT_SET_TIMES": 8388608,
+        "__WASI_RIGHTS_FD_READ": 2,
+        "__WASI_RIGHTS_FD_READDIR": 16384,
+        "__WASI_RIGHTS_FD_SEEK": 4,
+        "__WASI_RIGHTS_FD_SYNC": 16,
+        "__WASI_RIGHTS_FD_TELL": 32,
+        "__WASI_RIGHTS_FD_WRITE": 64,
+        "__WASI_RIGHTS_PATH_CREATE_DIRECTORY": 512,
+        "__WASI_RIGHTS_PATH_CREATE_FILE": 1024,
+        "__WASI_RIGHTS_PATH_FILESTAT_GET": 262144,
+        "__WASI_RIGHTS_PATH_FILESTAT_SET_SIZE": 524288,
+        "__WASI_RIGHTS_PATH_FILESTAT_SET_TIMES": 1048576,
+        "__WASI_RIGHTS_PATH_LINK_SOURCE": 2048,
+        "__WASI_RIGHTS_PATH_LINK_TARGET": 4096,
+        "__WASI_RIGHTS_PATH_OPEN": 8192,
+        "__WASI_RIGHTS_PATH_READLINK": 32768,
+        "__WASI_RIGHTS_PATH_REMOVE_DIRECTORY": 33554432,
+        "__WASI_RIGHTS_PATH_RENAME_SOURCE": 65536,
+        "__WASI_RIGHTS_PATH_RENAME_TARGET": 131072,
+        "__WASI_RIGHTS_PATH_SYMLINK": 16777216,
+        "__WASI_RIGHTS_PATH_UNLINK_FILE": 67108864,
+        "__WASI_RIGHTS_POLL_FD_READWRITE": 134217728,
+        "__WASI_RIGHTS_SOCK_SHUTDOWN": 268435456
     },
     "structs": {
         "EmscriptenBatteryEvent": {
@@ -1236,6 +1275,26 @@
             "fs_rights_base": 8,
             "fs_rights_inheriting": 16
         },
+        "__wasi_filestat_t": {
+            "__size__": 64,
+            "atim": 40,
+            "ctim": 56,
+            "dev": 0,
+            "filetype": 16,
+            "ino": 8,
+            "mtim": 48,
+            "nlink": 24,
+            "size": 32
+        },
+        "__wasi_prestat_dir_t": {
+            "__size__": 8,
+            "pr_name_len": 0
+        },
+        "__wasi_prestat_t": {
+            "__size__": 16,
+            "pr_type": 0,
+            "u": 8
+        },
         "addrinfo": {
             "__size__": 48,
             "ai_addr": 24,

src/library_sigs.js

@@ -1001,9 +1001,11 @@ sigs = {
   lineRGBA__sig: 'ipiiiiiiii',
   pixelRGBA__sig: 'ipiiiiii',
   proc_exit__sig: 'vi',
+  random_get__sig: 'ipp',
   rectangleColor__sig: 'ipiiiii',
   rectangleRGBA__sig: 'ipiiiiiiii',
   rotozoomSurface__sig: 'ppddi',
+  sched_yield__sig: 'i',
   setprotoent__sig: 'vi',
   strftime__sig: 'ppppp',
   strftime_l__sig: 'pppppp',

src/library_syscall.js

@@ -994,7 +994,18 @@ function wrapSyscallFunction(x, library, isWasi) {
   var canThrow = library[x + '__nothrow'] !== true;
 #endif
 
-  var pre = '', post = '';
+  if (!library[x + '__deps']) library[x + '__deps'] = [];
+
+#if PURE_WASI
+  // In PURE_WASI mode we can't assume the wasm binary was built by emscripten
+  // and politely notify us on memory growth.  Instead we have to check for
+  // possible memory growth on each syscall.
+  var pre = '\nif (!HEAPU8.byteLength) _emscripten_notify_memory_growth(0);\n'
+  library[x + '__deps'].push('emscripten_notify_memory_growth');
+#else
+  var pre = '';
+#endif
+  var post = '';
   if (isVariadic) {
     pre += 'SYSCALLS.varargs = varargs;\n';
   }
@@ -1047,7 +1058,6 @@ function wrapSyscallFunction(x, library, isWasi) {
   }
 
   library[x] = eval('(' + t + ')');
-  if (!library[x + '__deps']) library[x + '__deps'] = [];
   library[x + '__deps'].push('$SYSCALLS');
 #if PTHREADS
   // Most syscalls need to happen on the main JS thread (e.g. because the